iris: some initial HiZ bits

7 files changed, 146 insertions, 17 deletions

diff --git a/src/gallium/drivers/iris/iris_blit.c b/src/gallium/drivers/iris/iris_blit.c
index 8b704d6d891..836c9e69132 100644
--- a/src/gallium/drivers/iris/iris_blit.c
+++ b/src/gallium/drivers/iris/iris_blit.c
@@ -224,10 +224,15 @@ void
 iris_blorp_surf_for_resource(struct blorp_surf *surf,
                              struct pipe_resource *p_res,
                              enum isl_aux_usage aux_usage,
+                             unsigned level,
                              bool is_render_target)
 {
    struct iris_resource *res = (void *) p_res;
 
+   if (aux_usage == ISL_AUX_USAGE_HIZ &&
+       !iris_resource_level_has_hiz(res, level))
+      aux_usage = ISL_AUX_USAGE_NONE;
+
    *surf = (struct blorp_surf) {
       .surf = &res->surf,
       .addr = (struct blorp_address) {
@@ -306,9 +311,9 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
 
    struct blorp_surf src_surf, dst_surf;
    iris_blorp_surf_for_resource(&src_surf, info->src.resource,
-                                src_aux_usage, false);
+                                src_aux_usage, info->src.level, false);
    iris_blorp_surf_for_resource(&dst_surf, info->dst.resource,
-                                dst_aux_usage, true);
+                                dst_aux_usage, info->dst.level, true);
 
    iris_resource_prepare_access(ice, batch, dst_res, info->dst.level, 1,
                                 info->dst.box.z, info->dst.box.depth,
@@ -414,9 +419,9 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
       iris_get_depth_stencil_resources(info->src.resource, &junk, &src_res);
       iris_get_depth_stencil_resources(info->dst.resource, &junk, &dst_res);
       iris_blorp_surf_for_resource(&src_surf, &src_res->base,
-                                   ISL_AUX_USAGE_NONE, false);
+                                   ISL_AUX_USAGE_NONE, info->src.level, false);
       iris_blorp_surf_for_resource(&dst_surf, &dst_res->base,
-                                   ISL_AUX_USAGE_NONE, true);
+                                   ISL_AUX_USAGE_NONE, info->dst.level, true);
 
       for (int slice = 0; slice < info->dst.box.depth; slice++) {
          iris_batch_maybe_flush(batch, 1500);
@@ -515,8 +520,10 @@ iris_resource_copy_region(struct pipe_context *ctx,
       // XXX: what about one surface being a buffer and not the other?
 
       struct blorp_surf src_surf, dst_surf;
-      iris_blorp_surf_for_resource(&src_surf, src, src_aux_usage, false);
-      iris_blorp_surf_for_resource(&dst_surf, dst, dst_aux_usage, true);
+      iris_blorp_surf_for_resource(&src_surf, src, src_aux_usage,
+                                   src_level, false);
+      iris_blorp_surf_for_resource(&dst_surf, dst, dst_aux_usage,
+                                   dst_level, true);
 
       iris_resource_prepare_access(ice, batch, src_res, src_level, 1,
                                    src_box->z, src_box->depth,
diff --git a/src/gallium/drivers/iris/iris_clear.c b/src/gallium/drivers/iris/iris_clear.c
index 11b0a55ba41..e913dc63e1a 100644
--- a/src/gallium/drivers/iris/iris_clear.c
+++ b/src/gallium/drivers/iris/iris_clear.c
@@ -72,7 +72,7 @@ clear_color(struct iris_context *ice,
                                 box->z, box->depth, aux_usage);
 
    struct blorp_surf surf;
-   iris_blorp_surf_for_resource(&surf, p_res, aux_usage, true);
+   iris_blorp_surf_for_resource(&surf, p_res, aux_usage, level, true);
 
    if (!isl_format_supports_rendering(devinfo, format) &&
        isl_format_is_rgbx(format))
@@ -130,12 +130,12 @@ clear_depth_stencil(struct iris_context *ice,
    if (z_res) {
       iris_resource_prepare_depth(ice, batch, z_res, level, box->z, box->depth);
       iris_blorp_surf_for_resource(&z_surf, &z_res->base,
-                                   z_res->aux.usage, true);
+                                   z_res->aux.usage, level, true);
    }
 
    if (stencil_res) {
       iris_blorp_surf_for_resource(&stencil_surf, &stencil_res->base,
-                                   stencil_res->aux.usage, true);
+                                   stencil_res->aux.usage, level, true);
    }
 
    blorp_clear_depth_stencil(&blorp_batch, &z_surf, &stencil_surf,
diff --git a/src/gallium/drivers/iris/iris_context.h b/src/gallium/drivers/iris/iris_context.h
index 4fa48abb457..8453e8c8d9e 100644
--- a/src/gallium/drivers/iris/iris_context.h
+++ b/src/gallium/drivers/iris/iris_context.h
@@ -611,6 +611,7 @@ void iris_fill_cs_push_const_buffer(struct brw_cs_prog_data *cs_prog_data,
 void iris_blorp_surf_for_resource(struct blorp_surf *surf,
                                   struct pipe_resource *p_res,
                                   enum isl_aux_usage aux_usage,
+                                  unsigned level,
                                   bool is_render_target);
 
 /* iris_draw.c */
diff --git a/src/gallium/drivers/iris/iris_resolve.c b/src/gallium/drivers/iris/iris_resolve.c
index 4c3b1bdfabb..b1d0856d8e8 100644
--- a/src/gallium/drivers/iris/iris_resolve.c
+++ b/src/gallium/drivers/iris/iris_resolve.c
@@ -408,7 +408,8 @@ iris_resolve_color(struct iris_context *ice,
    //DBG("%s to mt %p level %u layer %u\n", __FUNCTION__, mt, level, layer);
 
    struct blorp_surf surf;
-   iris_blorp_surf_for_resource(&surf, &res->base, res->aux.usage, true);
+   iris_blorp_surf_for_resource(&surf, &res->base, res->aux.usage, level,
+                                true);
 
    iris_batch_maybe_flush(batch, 1500);
 
@@ -450,7 +451,7 @@ iris_mcs_partial_resolve(struct iris_context *ice,
    assert(res->aux.usage == ISL_AUX_USAGE_MCS);
 
    struct blorp_surf surf;
-   iris_blorp_surf_for_resource(&surf, &res->base, res->aux.usage, true);
+   iris_blorp_surf_for_resource(&surf, &res->base, res->aux.usage, 0, true);
 
    struct blorp_batch blorp_batch;
    blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
@@ -516,14 +517,114 @@ sample_with_hiz(const struct gen_device_info *devinfo,
 }
 
 /**
+ * Perform a HiZ or depth resolve operation.
+ *
+ * For an overview of HiZ ops, see the following sections of the Sandy Bridge
+ * PRM, Volume 1, Part 2:
+ *   - 7.5.3.1 Depth Buffer Clear
+ *   - 7.5.3.2 Depth Buffer Resolve
+ *   - 7.5.3.3 Hierarchical Depth Buffer Resolve
+ */
+static void
+iris_hiz_exec(struct iris_context *ice,
+              struct iris_batch *batch,
+              struct iris_resource *res,
+              unsigned int level, unsigned int start_layer,
+              unsigned int num_layers, enum isl_aux_op op)
+{
+   assert(iris_resource_level_has_hiz(res, level));
+   assert(op != ISL_AUX_OP_NONE);
+   const char *name = NULL;
+
+   switch (op) {
+   case ISL_AUX_OP_FULL_RESOLVE:
+      name = "depth resolve";
+      break;
+   case ISL_AUX_OP_AMBIGUATE:
+      name = "hiz ambiguate";
+      break;
+   case ISL_AUX_OP_FAST_CLEAR:
+      name = "depth clear";
+      break;
+   case ISL_AUX_OP_PARTIAL_RESOLVE:
+   case ISL_AUX_OP_NONE:
+      unreachable("Invalid HiZ op");
+   }
+
+   //DBG("%s %s to mt %p level %d layers %d-%d\n",
+       //__func__, name, mt, level, start_layer, start_layer + num_layers - 1);
+
+   /* The following stalls and flushes are only documented to be required
+    * for HiZ clear operations.  However, they also seem to be required for
+    * resolve operations.
+    *
+    * From the Ivybridge PRM, volume 2, "Depth Buffer Clear":
+    *
+    *   "If other rendering operations have preceded this clear, a
+    *    PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
+    *    enabled must be issued before the rectangle primitive used for
+    *    the depth buffer clear operation."
+    *
+    * Same applies for Gen8 and Gen9.
+    *
+    * In addition, from the Ivybridge PRM, volume 2, 1.10.4.1
+    * PIPE_CONTROL, Depth Cache Flush Enable:
+    *
+    *   "This bit must not be set when Depth Stall Enable bit is set in
+    *    this packet."
+    *
+    * This is confirmed to hold for real, Haswell gets immediate gpu hangs.
+    *
+    * Therefore issue two pipe control flushes, one for cache flush and
+    * another for depth stall.
+    */
+   iris_emit_pipe_control_flush(batch,
+                                PIPE_CONTROL_DEPTH_CACHE_FLUSH |
+                                PIPE_CONTROL_CS_STALL);
+
+   iris_emit_pipe_control_flush(batch, PIPE_CONTROL_DEPTH_STALL);
+
+   assert(res->aux.usage == ISL_AUX_USAGE_HIZ && res->aux.bo);
+
+   struct blorp_surf surf;
+   iris_blorp_surf_for_resource(&surf, &res->base, ISL_AUX_USAGE_HIZ,
+                                level, true);
+
+   struct blorp_batch blorp_batch;
+   blorp_batch_init(&ice->blorp, &blorp_batch, batch,
+                    BLORP_BATCH_NO_UPDATE_CLEAR_COLOR);
+   blorp_hiz_op(&blorp_batch, &surf, level, start_layer, num_layers, op);
+   blorp_batch_finish(&blorp_batch);
+
+   /* The following stalls and flushes are only documented to be required
+    * for HiZ clear operations.  However, they also seem to be required for
+    * resolve operations.
+    *
+    * From the Broadwell PRM, volume 7, "Depth Buffer Clear":
+    *
+    *    "Depth buffer clear pass using any of the methods (WM_STATE,
+    *     3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a
+    *     PIPE_CONTROL command with DEPTH_STALL bit and Depth FLUSH bits
+    *     "set" before starting to render.  DepthStall and DepthFlush are
+    *     not needed between consecutive depth clear passes nor is it
+    *     required if the depth clear pass was done with
+    *     'full_surf_clear' bit set in the 3DSTATE_WM_HZ_OP."
+    *
+    * TODO: Such as the spec says, this could be conditional.
+    */
+   iris_emit_pipe_control_flush(batch,
+                                PIPE_CONTROL_DEPTH_CACHE_FLUSH |
+                                PIPE_CONTROL_DEPTH_STALL);
+}
+
+/**
  * Does the resource's slice have hiz enabled?
  */
 bool
 iris_resource_level_has_hiz(const struct iris_resource *res, uint32_t level)
 {
    iris_resource_check_level_layer(res, level, 0);
-   // return res->level[level].has_hiz;
-   return false;
+   return res->aux.has_hiz & 1 << level;
 }
 
 /** \brief Assert that the level and layer are valid for the resource. */
@@ -887,8 +988,7 @@ iris_resource_prepare_hiz_access(struct iris_context *ice,
    }
 
    if (hiz_op != ISL_AUX_OP_NONE) {
-      // XXX: HiZ
-      //intel_hiz_exec(ice, res, level, layer, 1, hiz_op);
+      iris_hiz_exec(ice, batch, res, level, layer, 1, hiz_op);
 
       switch (hiz_op) {
       case ISL_AUX_OP_FULL_RESOLVE:
diff --git a/src/gallium/drivers/iris/iris_resource.c b/src/gallium/drivers/iris/iris_resource.c
index badf29542e3..2f6a0c61bf2 100644
--- a/src/gallium/drivers/iris/iris_resource.c
+++ b/src/gallium/drivers/iris/iris_resource.c
@@ -203,7 +203,6 @@ iris_resource_disable_aux(struct iris_resource *res)
    free(res->aux.state);
 
    // XXX: clear color BO
-   // XXX: HiZ
 
    res->aux.usage = ISL_AUX_USAGE_NONE;
    res->aux.possible_usages = 1 << ISL_AUX_USAGE_NONE;
@@ -376,7 +375,18 @@ iris_resource_alloc_aux(struct iris_screen *screen, struct iris_resource *res)
       iris_bo_unmap(res->aux.bo);
    }
 
-   // XXX: HIZ enabling
+   if (res->aux.usage == ISL_AUX_USAGE_HIZ) {
+      for (unsigned level = 0; level < res->surf.levels; ++level) {
+         uint32_t width = u_minify(res->surf.phys_level0_sa.width, level);
+         uint32_t height = u_minify(res->surf.phys_level0_sa.height, level);
+
+         /* Disable HiZ for LOD > 0 unless the width/height are 8x4 aligned.
+          * For LOD == 0, we can grow the dimensions to make it work.
+          */
+         if (level == 0 || ((width & 7) == 0 && (height & 3) == 0))
+            res->aux.has_hiz |= 1 << level;
+      }
+   }
 
    return true;
 }
diff --git a/src/gallium/drivers/iris/iris_resource.h b/src/gallium/drivers/iris/iris_resource.h
index 382f1159c5a..26451cbed4b 100644
--- a/src/gallium/drivers/iris/iris_resource.h
+++ b/src/gallium/drivers/iris/iris_resource.h
@@ -111,6 +111,11 @@ struct iris_resource {
        * aux state for each slice.
        */
       enum isl_aux_state **state;
+
+      /**
+       * If (1 << level) is set, HiZ is enabled for that miplevel.
+       */
+      uint16_t has_hiz;
    } aux;
 
    /**
diff --git a/src/gallium/drivers/iris/iris_state.c b/src/gallium/drivers/iris/iris_state.c
index 0cdfbd2af89..bd511c9645d 100644
--- a/src/gallium/drivers/iris/iris_state.c
+++ b/src/gallium/drivers/iris/iris_state.c
@@ -2268,6 +2268,12 @@ iris_set_framebuffer_state(struct pipe_context *ctx,
          info.mocs = mocs(zres->bo);
 
          view.format = zres->surf.format;
+
+         if (iris_resource_level_has_hiz(zres, view.base_level)) {
+            info.hiz_usage = ISL_AUX_USAGE_HIZ;
+            info.hiz_surf = &zres->aux.surf;
+            info.hiz_address = zres->aux.bo->gtt_offset;
+         }
       }
 
       if (stencil_res) {