ilo: resolve Z/HiZ correctly

When the depth buffer is to be read, perform a Depth Buffer Resolve if it has
been rendered.  When the depth buffer is to be rendered, perform a HiZ Buffer
Resolve when the depth buffer is modified externally.

5 files changed, 234 insertions, 1 deletions

diff --git a/src/gallium/drivers/ilo/ilo_3d.c b/src/gallium/drivers/ilo/ilo_3d.c
index 5f0f2312f75..63fec036bc6 100644
--- a/src/gallium/drivers/ilo/ilo_3d.c
+++ b/src/gallium/drivers/ilo/ilo_3d.c
@@ -29,6 +29,7 @@
 #include "intel_winsys.h"
 
 #include "ilo_3d_pipeline.h"
+#include "ilo_blit.h"
 #include "ilo_context.h"
 #include "ilo_cp.h"
 #include "ilo_query.h"
@@ -737,6 +738,8 @@ ilo_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info)
    if (!upload_shaders(hw3d, ilo->shader_cache))
       return;
 
+   ilo_blit_resolve_framebuffer(ilo);
+
    /* If draw_vbo ever fails, return immediately. */
    if (!draw_vbo(hw3d, ilo, &prim_generated, &prim_emitted))
       return;
diff --git a/src/gallium/drivers/ilo/ilo_blit.c b/src/gallium/drivers/ilo/ilo_blit.c
index 692224932b3..17193ab7dc6 100644
--- a/src/gallium/drivers/ilo/ilo_blit.c
+++ b/src/gallium/drivers/ilo/ilo_blit.c
@@ -129,8 +129,71 @@ ilo_blit(struct pipe_context *pipe, const struct pipe_blit_info *info)
 }
 
 static void
-ilo_flush_resource(struct pipe_context *ctx, struct pipe_resource *resource)
+ilo_flush_resource(struct pipe_context *pipe, struct pipe_resource *res)
 {
+   struct ilo_context *ilo = ilo_context(pipe);
+   const unsigned flags = ILO_TEXTURE_CPU_READ |
+                          ILO_TEXTURE_BLT_READ |
+                          ILO_TEXTURE_RENDER_READ;
+
+   ilo_blit_resolve_resource(ilo, res, flags);
+}
+
+void
+ilo_blit_resolve_slices_for_hiz(struct ilo_context *ilo,
+                                struct pipe_resource *res, unsigned level,
+                                unsigned first_slice, unsigned num_slices,
+                                unsigned flags)
+{
+   struct ilo_texture *tex = ilo_texture(res);
+   const unsigned any_reader =
+      ILO_TEXTURE_RENDER_READ |
+      ILO_TEXTURE_BLT_READ |
+      ILO_TEXTURE_CPU_READ;
+   const unsigned other_writers =
+      ILO_TEXTURE_BLT_WRITE |
+      ILO_TEXTURE_CPU_WRITE;
+   unsigned i;
+
+   assert(tex->base.target != PIPE_BUFFER && tex->hiz.bo);
+
+   if (flags & ILO_TEXTURE_RENDER_WRITE) {
+      /*
+       * When ILO_TEXTURE_RENDER_WRITE is set, there can be no reader.  We
+       * need to perform a HiZ Buffer Resolve in case the resource was
+       * previously written by another writer, unless this is a clear.
+       */
+      assert(!(flags & (other_writers | any_reader)));
+
+      if (!(flags & ILO_TEXTURE_CLEAR)) {
+         for (i = 0; i < num_slices; i++) {
+            const struct ilo_texture_slice *slice =
+               ilo_texture_get_slice(tex, level, first_slice + i);
+
+            if (slice->flags & other_writers) {
+               ilo_blitter_rectlist_resolve_hiz(ilo->blitter,
+                     res, level, first_slice + i);
+            }
+         }
+      }
+   }
+   else if ((flags & any_reader) ||
+         ((flags & other_writers) && !(flags & ILO_TEXTURE_CLEAR))) {
+      /*
+       * When there is at least a reader or writer, we need to perform a
+       * Depth Buffer Resolve in case the resource was previously written
+       * by ILO_TEXTURE_RENDER_WRITE.
+       */
+      for (i = 0; i < num_slices; i++) {
+         const struct ilo_texture_slice *slice =
+            ilo_texture_get_slice(tex, level, first_slice + i);
+
+         if (slice->flags & ILO_TEXTURE_RENDER_WRITE) {
+            ilo_blitter_rectlist_resolve_z(ilo->blitter,
+                  &tex->base, level, first_slice + i);
+         }
+      }
+   }
 }
 
 /**
diff --git a/src/gallium/drivers/ilo/ilo_blit.h b/src/gallium/drivers/ilo/ilo_blit.h
index 709af3fbed6..6e85f79724d 100644
--- a/src/gallium/drivers/ilo/ilo_blit.h
+++ b/src/gallium/drivers/ilo/ilo_blit.h
@@ -29,10 +29,164 @@
 #define ILO_BLIT_H
 
 #include "ilo_common.h"
+#include "ilo_context.h"
+#include "ilo_gpe.h"
+#include "ilo_resource.h"
 
 struct ilo_context;
 
 void
+ilo_blit_resolve_slices_for_hiz(struct ilo_context *ilo,
+                                struct pipe_resource *res, unsigned level,
+                                unsigned first_slice, unsigned num_slices,
+                                unsigned flags);
+
+static inline void
+ilo_blit_resolve_slices(struct ilo_context *ilo,
+                        struct pipe_resource *res, unsigned level,
+                        unsigned first_slice, unsigned num_slices,
+                        unsigned flags)
+{
+   struct ilo_texture *tex;
+   unsigned flag_mask;
+
+   if (res->target == PIPE_BUFFER)
+      return;
+
+   tex = ilo_texture(res);
+
+   /*
+    * This function is called frequently and we need to make it run faster.
+    * As it is only used to resolve HiZ right now, return early when there is
+    * no HiZ.
+    */
+   if (!tex->hiz.bo)
+      return;
+
+   /*
+    * flags may be
+    *
+    *  - ILO_TEXTURE_CPU_{READ,WRITE} (transfer)
+    *  - ILO_TEXTURE_BLT_{READ,WRITE} (BLT copy or clear)
+    *  - ILO_TEXTURE_RENDER_{READ,WRITE} (sample or render)
+    *  - ILO_TEXTURE_CLEAR
+    *
+    * It is assumed there is at most one writer, and that readers read before
+    * writers write.
+    */
+   if (tex->hiz.bo) {
+      ilo_blit_resolve_slices_for_hiz(ilo, res, level,
+            first_slice, num_slices, flags);
+   }
+
+   /* clear writers and clear state that are not set */
+   flag_mask =
+      ILO_TEXTURE_CPU_WRITE |
+      ILO_TEXTURE_BLT_WRITE |
+      ILO_TEXTURE_RENDER_WRITE;
+   if (flags & flag_mask)
+      flag_mask |= ILO_TEXTURE_CLEAR;
+
+   ilo_texture_set_slice_flags(tex, level,
+         first_slice, num_slices, flag_mask, flags);
+}
+
+static inline void
+ilo_blit_resolve_resource(struct ilo_context *ilo,
+                          struct pipe_resource *res,
+                          unsigned flags)
+{
+   unsigned lv;
+
+   for (lv = 0; lv <= res->last_level; lv++) {
+      const unsigned num_slices = (res->target == PIPE_TEXTURE_3D) ?
+         u_minify(res->depth0, lv) : res->array_size;
+
+      ilo_blit_resolve_slices(ilo, res, lv, 0, num_slices, flags);
+   }
+}
+
+static inline void
+ilo_blit_resolve_surface(struct ilo_context *ilo,
+                         struct pipe_surface *surf,
+                         unsigned flags)
+{
+   if (surf->texture->target == PIPE_BUFFER)
+      return;
+
+   ilo_blit_resolve_slices(ilo, surf->texture,
+         surf->u.tex.level, surf->u.tex.first_layer,
+         surf->u.tex.last_layer - surf->u.tex.first_layer + 1,
+         flags);
+}
+
+static inline void
+ilo_blit_resolve_transfer(struct ilo_context *ilo,
+                          const struct pipe_transfer *xfer)
+{
+   unsigned flags = 0;
+
+   if (xfer->resource->target == PIPE_BUFFER)
+      return;
+
+   if (xfer->usage & PIPE_TRANSFER_READ)
+      flags |= ILO_TEXTURE_CPU_READ;
+   if (xfer->usage & PIPE_TRANSFER_WRITE)
+      flags |= ILO_TEXTURE_CPU_WRITE;
+
+   ilo_blit_resolve_slices(ilo, xfer->resource, xfer->level,
+         xfer->box.z, xfer->box.depth, flags);
+}
+
+static inline void
+ilo_blit_resolve_view(struct ilo_context *ilo,
+                      const struct pipe_sampler_view *view)
+{
+   const unsigned flags = ILO_TEXTURE_RENDER_READ;
+   unsigned lv;
+
+   if (view->texture->target == PIPE_BUFFER)
+      return;
+
+   for (lv = view->u.tex.first_level; lv <= view->u.tex.last_level; lv++) {
+      unsigned first_slice, num_slices;
+
+      if (view->texture->target == PIPE_TEXTURE_3D) {
+         first_slice = 0;
+         num_slices = u_minify(view->texture->depth0, lv);
+      }
+      else {
+         first_slice = view->u.tex.first_layer;
+         num_slices = view->u.tex.last_layer - view->u.tex.first_layer + 1;
+      }
+
+      ilo_blit_resolve_slices(ilo, view->texture,
+            lv, first_slice, num_slices, flags);
+   }
+}
+
+static inline void
+ilo_blit_resolve_framebuffer(struct ilo_context *ilo)
+{
+   const struct pipe_framebuffer_state *fb = &ilo->fb.state;
+   unsigned sh, i;
+
+   /* Not all bound views are sampled by the shaders.  How do we tell? */
+   for (sh = 0; sh < Elements(ilo->view); sh++) {
+      for (i = 0; i < ilo->view[sh].count; i++) {
+         if (ilo->view[sh].states[i])
+            ilo_blit_resolve_view(ilo, ilo->view[sh].states[i]);
+      }
+   }
+
+   for (i = 0; i < fb->nr_cbufs; i++)
+      ilo_blit_resolve_surface(ilo, fb->cbufs[i], ILO_TEXTURE_RENDER_WRITE);
+
+   if (fb->zsbuf)
+      ilo_blit_resolve_surface(ilo, fb->zsbuf, ILO_TEXTURE_RENDER_WRITE);
+}
+
+void
 ilo_init_blit_functions(struct ilo_context *ilo);
 
 #endif /* ILO_BLIT_H */
diff --git a/src/gallium/drivers/ilo/ilo_blitter_blt.c b/src/gallium/drivers/ilo/ilo_blitter_blt.c
index dba84b0b747..d978150943c 100644
--- a/src/gallium/drivers/ilo/ilo_blitter_blt.c
+++ b/src/gallium/drivers/ilo/ilo_blitter_blt.c
@@ -31,6 +31,7 @@
 #include "ilo_3d.h"
 #include "ilo_context.h"
 #include "ilo_cp.h"
+#include "ilo_blit.h"
 #include "ilo_resource.h"
 #include "ilo_blitter.h"
 
@@ -654,6 +655,11 @@ ilo_blitter_blt_copy_resource(struct ilo_blitter *blitter,
 {
    bool success;
 
+   ilo_blit_resolve_slices(blitter->ilo, src, src_level,
+         src_box->z, src_box->depth, ILO_TEXTURE_BLT_READ);
+   ilo_blit_resolve_slices(blitter->ilo, dst, dst_level,
+         dst_z, src_box->depth, ILO_TEXTURE_BLT_WRITE);
+
    if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
       const unsigned dst_offset = dst_x;
       const unsigned src_offset = src_box->x;
@@ -715,6 +721,8 @@ ilo_blitter_blt_clear_rt(struct ilo_blitter *blitter,
        util_format_is_compressed(rt->format))
       return false;
 
+   ilo_blit_resolve_surface(blitter->ilo, rt, ILO_TEXTURE_BLT_WRITE);
+
    util_pack_color(color->f, rt->format, &packed);
 
    if (rt->texture->target == PIPE_BUFFER) {
@@ -800,6 +808,8 @@ ilo_blitter_blt_clear_zs(struct ilo_blitter *blitter,
       break;
    }
 
+   ilo_blit_resolve_surface(blitter->ilo, zs, ILO_TEXTURE_BLT_WRITE);
+
    val = util_pack_z_stencil(zs->format, depth, stencil);
 
    u_box_3d(x, y, zs->u.tex.first_layer, width, height,
diff --git a/src/gallium/drivers/ilo/ilo_transfer.c b/src/gallium/drivers/ilo/ilo_transfer.c
index 4bd688d5c73..5b8a6588e94 100644
--- a/src/gallium/drivers/ilo/ilo_transfer.c
+++ b/src/gallium/drivers/ilo/ilo_transfer.c
@@ -29,6 +29,7 @@
 #include "util/u_transfer.h"
 #include "util/u_format_etc.h"
 
+#include "ilo_blit.h"
 #include "ilo_cp.h"
 #include "ilo_context.h"
 #include "ilo_resource.h"
@@ -997,6 +998,8 @@ ilo_transfer_map(struct pipe_context *pipe,
    xfer->base.usage = usage;
    xfer->base.box = *box;
 
+   ilo_blit_resolve_transfer(ilo, &xfer->base);
+
    if (res->target == PIPE_BUFFER)
       success = buf_map(ilo, xfer);
    else