st/mesa: implement accelerated stencil blitting using shader stencil export

Reviewed-by: Alex Deucher <[email protected]>

4 files changed, 176 insertions, 38 deletions

diff --git a/src/gallium/auxiliary/postprocess/pp_mlaa.c b/src/gallium/auxiliary/postprocess/pp_mlaa.c
index 80db86227ff..951b76f0e89 100644
--- a/src/gallium/auxiliary/postprocess/pp_mlaa.c
+++ b/src/gallium/auxiliary/postprocess/pp_mlaa.c
@@ -179,7 +179,7 @@ pp_jimenezmlaa_run(struct pp_queue_t *ppq, struct pipe_resource *in,
    util_blit_pixels(p->blitctx, in, 0, 0, 0,
                     w, h, 0, p->framebuffer.cbufs[0],
                     0, 0, w, h, 0, PIPE_TEX_MIPFILTER_NEAREST,
-                    TGSI_WRITEMASK_XYZW);
+                    TGSI_WRITEMASK_XYZW, 0);
 
    u_sampler_view_default_template(&v_tmp, in, in->format);
    arr[0] = p->pipe->create_sampler_view(p->pipe, in, &v_tmp);
diff --git a/src/gallium/auxiliary/postprocess/pp_run.c b/src/gallium/auxiliary/postprocess/pp_run.c
index 9d0bfdf81ff..d2f7aa6dafb 100644
--- a/src/gallium/auxiliary/postprocess/pp_run.c
+++ b/src/gallium/auxiliary/postprocess/pp_run.c
@@ -60,7 +60,7 @@ pp_run(struct pp_queue_t *ppq, struct pipe_resource *in,
       util_blit_pixels(ppq->p->blitctx, in, 0, 0, 0,
                        w, h, 0, ppq->tmps[0],
                        0, 0, w, h, 0, PIPE_TEX_MIPFILTER_NEAREST,
-                       TGSI_WRITEMASK_XYZW);
+                       TGSI_WRITEMASK_XYZW, 0);
 
       in = ppq->tmp[0];
    }
diff --git a/src/gallium/auxiliary/util/u_blit.c b/src/gallium/auxiliary/util/u_blit.c
index 30f4a93d2dc..b52f4c092c2 100644
--- a/src/gallium/auxiliary/util/u_blit.c
+++ b/src/gallium/auxiliary/util/u_blit.c
@@ -57,8 +57,10 @@ struct blit_state
    struct cso_context *cso;
 
    struct pipe_blend_state blend_write_color, blend_keep_color;
-   struct pipe_depth_stencil_alpha_state depthstencil_keep;
-   struct pipe_depth_stencil_alpha_state depthstencil_write;
+   struct pipe_depth_stencil_alpha_state dsa_keep_depthstencil;
+   struct pipe_depth_stencil_alpha_state dsa_write_depthstencil;
+   struct pipe_depth_stencil_alpha_state dsa_write_depth;
+   struct pipe_depth_stencil_alpha_state dsa_write_stencil;
    struct pipe_rasterizer_state rasterizer;
    struct pipe_sampler_state sampler;
    struct pipe_viewport_state viewport;
@@ -67,12 +69,16 @@ struct blit_state
 
    void *vs;
    void *fs[PIPE_MAX_TEXTURE_TYPES][TGSI_WRITEMASK_XYZW + 1];
+   void *fs_depthstencil[PIPE_MAX_TEXTURE_TYPES];
    void *fs_depth[PIPE_MAX_TEXTURE_TYPES];
+   void *fs_stencil[PIPE_MAX_TEXTURE_TYPES];
 
    struct pipe_resource *vbuf;  /**< quad vertices */
    unsigned vbuf_slot;
 
    float vertices[4][2][4];   /**< vertex/texcoords for quad */
+
+   boolean has_stencil_export;
 };
 
 
@@ -96,10 +102,19 @@ util_create_blit(struct pipe_context *pipe, struct cso_context *cso)
    /* disabled blending/masking */
    ctx->blend_write_color.rt[0].colormask = PIPE_MASK_RGBA;
 
-   /* no-op depth/stencil/alpha */
-   ctx->depthstencil_write.depth.enabled = 1;
-   ctx->depthstencil_write.depth.writemask = 1;
-   ctx->depthstencil_write.depth.func = PIPE_FUNC_ALWAYS;
+   /* depth stencil states */
+   ctx->dsa_write_depth.depth.enabled = 1;
+   ctx->dsa_write_depth.depth.writemask = 1;
+   ctx->dsa_write_depth.depth.func = PIPE_FUNC_ALWAYS;
+   ctx->dsa_write_stencil.stencil[0].enabled = 1;
+   ctx->dsa_write_stencil.stencil[0].func = PIPE_FUNC_ALWAYS;
+   ctx->dsa_write_stencil.stencil[0].fail_op = PIPE_STENCIL_OP_REPLACE;
+   ctx->dsa_write_stencil.stencil[0].zpass_op = PIPE_STENCIL_OP_REPLACE;
+   ctx->dsa_write_stencil.stencil[0].zfail_op = PIPE_STENCIL_OP_REPLACE;
+   ctx->dsa_write_stencil.stencil[0].valuemask = 0xff;
+   ctx->dsa_write_stencil.stencil[0].writemask = 0xff;
+   ctx->dsa_write_depthstencil.depth = ctx->dsa_write_depth.depth;
+   ctx->dsa_write_depthstencil.stencil[0] = ctx->dsa_write_stencil.stencil[0];
 
    /* rasterizer */
    ctx->rasterizer.cull_face = PIPE_FACE_NONE;
@@ -136,6 +151,9 @@ util_create_blit(struct pipe_context *pipe, struct cso_context *cso)
    else
       ctx->internal_target = PIPE_TEXTURE_RECT;
 
+   ctx->has_stencil_export =
+      pipe->screen->get_param(pipe->screen, PIPE_CAP_SHADER_STENCIL_EXPORT);
+
    return ctx;
 }
 
@@ -159,10 +177,16 @@ util_destroy_blit(struct blit_state *ctx)
       }
    }
 
-   for (i = 0; i < Elements(ctx->fs_depth); i++) {
+   for (i = 0; i < PIPE_MAX_TEXTURE_TYPES; i++) {
+      if (ctx->fs_depthstencil[i]) {
+         pipe->delete_fs_state(pipe, ctx->fs_depthstencil[i]);
+      }
       if (ctx->fs_depth[i]) {
          pipe->delete_fs_state(pipe, ctx->fs_depth[i]);
       }
+      if (ctx->fs_stencil[i]) {
+         pipe->delete_fs_state(pipe, ctx->fs_stencil[i]);
+      }
    }
 
    pipe_resource_reference(&ctx->vbuf, NULL);
@@ -192,7 +216,26 @@ set_fragment_shader(struct blit_state *ctx, uint writemask,
 
 
 /**
- * Helper function to set the depthwrite shader.
+ * Helper function to set the shader which writes depth and stencil.
+ */
+static INLINE void
+set_depthstencil_fragment_shader(struct blit_state *ctx,
+                                 enum pipe_texture_target pipe_tex)
+{
+   if (!ctx->fs_depthstencil[pipe_tex]) {
+      unsigned tgsi_tex = util_pipe_tex_to_tgsi_tex(pipe_tex);
+
+      ctx->fs_depthstencil[pipe_tex] =
+         util_make_fragment_tex_shader_writedepthstencil(ctx->pipe, tgsi_tex,
+                                                  TGSI_INTERPOLATE_LINEAR);
+   }
+
+   cso_set_fragment_shader_handle(ctx->cso, ctx->fs_depthstencil[pipe_tex]);
+}
+
+
+/**
+ * Helper function to set the shader which writes depth.
  */
 static INLINE void
 set_depth_fragment_shader(struct blit_state *ctx,
@@ -211,6 +254,25 @@ set_depth_fragment_shader(struct blit_state *ctx,
 
 
 /**
+ * Helper function to set the shader which writes stencil.
+ */
+static INLINE void
+set_stencil_fragment_shader(struct blit_state *ctx,
+                            enum pipe_texture_target pipe_tex)
+{
+   if (!ctx->fs_stencil[pipe_tex]) {
+      unsigned tgsi_tex = util_pipe_tex_to_tgsi_tex(pipe_tex);
+
+      ctx->fs_stencil[pipe_tex] =
+         util_make_fragment_tex_shader_writestencil(ctx->pipe, tgsi_tex,
+                                                    TGSI_INTERPOLATE_LINEAR);
+   }
+
+   cso_set_fragment_shader_handle(ctx->cso, ctx->fs_stencil[pipe_tex]);
+}
+
+
+/**
  * Helper function to set the vertex shader.
  */
 static INLINE void
@@ -358,7 +420,6 @@ formats_compatible(enum pipe_format src_format,
  * \param writemask  controls which channels in the dest surface are sourced
  *                   from the src surface.  Disabled channels are sourced
  *                   from (0,0,0,1).
- * XXX need some control over blitting stencil.
  */
 void
 util_blit_pixels(struct blit_state *ctx,
@@ -371,7 +432,7 @@ util_blit_pixels(struct blit_state *ctx,
                  int dstX0, int dstY0,
                  int dstX1, int dstY1,
                  float z, uint filter,
-                 uint writemask)
+                 uint writemask, uint zs_writemask)
 {
    struct pipe_context *pipe = ctx->pipe;
    struct pipe_screen *screen = pipe->screen;
@@ -383,9 +444,12 @@ util_blit_pixels(struct blit_state *ctx,
    const int srcW = abs(srcX1 - srcX0);
    const int srcH = abs(srcY1 - srcY0);
    unsigned offset;
-   boolean overlap, dst_is_depth;
+   boolean overlap;
    float s0, t0, s1, t1;
    boolean normalized;
+   boolean is_stencil, is_depth, blit_depth, blit_stencil;
+   const struct util_format_description *src_desc =
+         util_format_description(src_tex->format);
 
    assert(filter == PIPE_TEX_MIPFILTER_NEAREST ||
           filter == PIPE_TEX_MIPFILTER_LINEAR);
@@ -402,12 +466,24 @@ util_blit_pixels(struct blit_state *ctx,
    src_format = util_format_linear(src_tex->format);
    dst_format = util_format_linear(dst->format);
 
+   /* See whether we will blit depth or stencil. */
+   is_depth = util_format_has_depth(src_desc);
+   is_stencil = util_format_has_stencil(src_desc);
+
+   blit_depth = is_depth && (zs_writemask & BLIT_WRITEMASK_Z);
+   blit_stencil = is_stencil && (zs_writemask & BLIT_WRITEMASK_STENCIL);
+
+   assert((writemask && !zs_writemask && !is_depth && !is_stencil) ||
+          (!writemask && (blit_depth || blit_stencil)));
+
    /*
     * Check for simple case:  no format conversion, no flipping, no stretching,
     * no overlapping.
     * Filter mode should not matter since there's no stretching.
     */
    if (formats_compatible(src_format, dst_format) &&
+       is_stencil == blit_stencil &&
+       is_depth == blit_depth &&
        srcX0 < srcX1 &&
        dstX0 < dstX1 &&
        srcY0 < srcY1 &&
@@ -430,6 +506,17 @@ util_blit_pixels(struct blit_state *ctx,
       return;
    }
 
+   /* It's a mistake to call this function with a stencil format and
+    * without shader stencil export. We don't do software fallbacks here.
+    * Ignore stencil and only copy depth.
+    */
+   if (blit_stencil && !ctx->has_stencil_export) {
+      blit_stencil = FALSE;
+
+      if (!blit_depth)
+         return;
+   }
+
    if (dst_format == dst->format) {
       dst_surface = dst;
    } else {
@@ -508,6 +595,11 @@ util_blit_pixels(struct blit_state *ctx,
       }
 
       u_sampler_view_default_template(&sv_templ, tex, tex->format);
+      if (!blit_depth && blit_stencil) {
+         /* set a stencil-only format, e.g. Z24S8 --> X24S8 */
+         sv_templ.format = util_format_stencil_only(tex->format);
+         assert(sv_templ.format != PIPE_FORMAT_NONE);
+      }
       sampler_view = pipe->create_sampler_view(pipe, tex, &sv_templ);
 
       if (!sampler_view) {
@@ -519,6 +611,11 @@ util_blit_pixels(struct blit_state *ctx,
    else {
       /* Directly sample from the source resource/texture */
       u_sampler_view_default_template(&sv_templ, src_tex, src_format);
+      if (!blit_depth && blit_stencil) {
+         /* set a stencil-only format, e.g. Z24S8 --> X24S8 */
+         sv_templ.format = util_format_stencil_only(src_format);
+         assert(sv_templ.format != PIPE_FORMAT_NONE);
+      }
       sampler_view = pipe->create_sampler_view(pipe, src_tex, &sv_templ);
 
       if (!sampler_view) {
@@ -539,15 +636,14 @@ util_blit_pixels(struct blit_state *ctx,
       }
    }
 
-   dst_is_depth = util_format_is_depth_or_stencil(dst_format);
-
-   assert(screen->is_format_supported(screen, sampler_view->format, ctx->internal_target,
-                                      sampler_view->texture->nr_samples,
-                                      PIPE_BIND_SAMPLER_VIEW));
+   assert(screen->is_format_supported(screen, sampler_view->format,
+                     ctx->internal_target, sampler_view->texture->nr_samples,
+                     PIPE_BIND_SAMPLER_VIEW));
    assert(screen->is_format_supported(screen, dst_format, ctx->internal_target,
-                                      dst_surface->texture->nr_samples,
-                                      dst_is_depth ? PIPE_BIND_DEPTH_STENCIL :
-                                                     PIPE_BIND_RENDER_TARGET));
+                     dst_surface->texture->nr_samples,
+                     is_depth || is_stencil ? PIPE_BIND_DEPTH_STENCIL :
+                                              PIPE_BIND_RENDER_TARGET));
+
    /* save state (restored below) */
    cso_save_blend(ctx->cso);
    cso_save_depth_stencil_alpha(ctx->cso);
@@ -569,22 +665,71 @@ util_blit_pixels(struct blit_state *ctx,
    else
       cso_set_blend(ctx->cso, &ctx->blend_keep_color);
 
-   cso_set_depth_stencil_alpha(ctx->cso,
-                               dst_is_depth ? &ctx->depthstencil_write :
-                                              &ctx->depthstencil_keep);
    cso_set_rasterizer(ctx->cso, &ctx->rasterizer);
    cso_set_vertex_elements(ctx->cso, 2, ctx->velem);
    cso_set_stream_outputs(ctx->cso, 0, NULL, 0);
 
-   /* sampler */
+   /* default sampler state */
    ctx->sampler.normalized_coords = normalized;
    ctx->sampler.min_img_filter = filter;
    ctx->sampler.mag_img_filter = filter;
    ctx->sampler.min_lod = src_level;
    ctx->sampler.max_lod = src_level;
-   cso_single_sampler(ctx->cso, 0, &ctx->sampler);
+
+   /* Depth stencil state, fragment shader and sampler setup depending on what
+    * we blit.
+    */
+   if (blit_depth && blit_stencil) {
+      cso_single_sampler(ctx->cso, 0, &ctx->sampler);
+      /* don't filter stencil */
+      ctx->sampler.min_img_filter = PIPE_TEX_FILTER_NEAREST;
+      ctx->sampler.mag_img_filter = PIPE_TEX_FILTER_NEAREST;
+      cso_single_sampler(ctx->cso, 1, &ctx->sampler);
+
+      cso_set_depth_stencil_alpha(ctx->cso, &ctx->dsa_write_depthstencil);
+      set_depthstencil_fragment_shader(ctx, sampler_view->texture->target);
+   }
+   else if (blit_depth) {
+      cso_single_sampler(ctx->cso, 0, &ctx->sampler);
+      cso_set_depth_stencil_alpha(ctx->cso, &ctx->dsa_write_depth);
+      set_depth_fragment_shader(ctx, sampler_view->texture->target);
+   }
+   else if (blit_stencil) {
+      /* don't filter stencil */
+      ctx->sampler.min_img_filter = PIPE_TEX_FILTER_NEAREST;
+      ctx->sampler.mag_img_filter = PIPE_TEX_FILTER_NEAREST;
+      cso_single_sampler(ctx->cso, 0, &ctx->sampler);
+
+      cso_set_depth_stencil_alpha(ctx->cso, &ctx->dsa_write_stencil);
+      set_stencil_fragment_shader(ctx, sampler_view->texture->target);
+   }
+   else { /* color */
+      cso_single_sampler(ctx->cso, 0, &ctx->sampler);
+      cso_set_depth_stencil_alpha(ctx->cso, &ctx->dsa_keep_depthstencil);
+      set_fragment_shader(ctx, writemask, sampler_view->texture->target);
+   }
    cso_single_sampler_done(ctx->cso);
 
+   /* textures */
+   if (blit_depth && blit_stencil) {
+      /* Setup two samplers, one for depth and the other one for stencil. */
+      struct pipe_sampler_view templ;
+      struct pipe_sampler_view *views[2];
+
+      templ = *sampler_view;
+      templ.format = util_format_stencil_only(templ.format);
+      assert(templ.format != PIPE_FORMAT_NONE);
+
+      views[0] = sampler_view;
+      views[1] = pipe->create_sampler_view(pipe, views[0]->texture, &templ);
+      cso_set_fragment_sampler_views(ctx->cso, 2, views);
+
+      pipe_sampler_view_reference(&views[1], NULL);
+   }
+   else {
+      cso_set_fragment_sampler_views(ctx->cso, 1, &sampler_view);
+   }
+
    /* viewport */
    ctx->viewport.scale[0] = 0.5f * dst_surface->width;
    ctx->viewport.scale[1] = 0.5f * dst_surface->height;
@@ -596,15 +741,6 @@ util_blit_pixels(struct blit_state *ctx,
    ctx->viewport.translate[3] = 0.0f;
    cso_set_viewport(ctx->cso, &ctx->viewport);
 
-   /* texture */
-   cso_set_fragment_sampler_views(ctx->cso, 1, &sampler_view);
-
-   /* shaders */
-   if (dst_is_depth) {
-      set_depth_fragment_shader(ctx, sampler_view->texture->target);
-   } else {
-      set_fragment_shader(ctx, writemask, sampler_view->texture->target);
-   }
    set_vertex_shader(ctx);
    cso_set_geometry_shader_handle(ctx->cso, NULL);
 
@@ -612,7 +748,7 @@ util_blit_pixels(struct blit_state *ctx,
    memset(&fb, 0, sizeof(fb));
    fb.width = dst_surface->width;
    fb.height = dst_surface->height;
-   if (dst_is_depth) {
+   if (blit_depth || blit_stencil) {
       fb.zsbuf = dst_surface;
    } else {
       fb.nr_cbufs = 1;
@@ -726,7 +862,7 @@ util_blit_pixels_tex(struct blit_state *ctx,
 
    /* set misc state we care about */
    cso_set_blend(ctx->cso, &ctx->blend_write_color);
-   cso_set_depth_stencil_alpha(ctx->cso, &ctx->depthstencil_keep);
+   cso_set_depth_stencil_alpha(ctx->cso, &ctx->dsa_keep_depthstencil);
    cso_set_rasterizer(ctx->cso, &ctx->rasterizer);
    cso_set_vertex_elements(ctx->cso, 2, ctx->velem);
    cso_set_stream_outputs(ctx->cso, 0, NULL, 0);
diff --git a/src/gallium/auxiliary/util/u_blit.h b/src/gallium/auxiliary/util/u_blit.h
index f26d429ace7..56ab0308305 100644
--- a/src/gallium/auxiliary/util/u_blit.h
+++ b/src/gallium/auxiliary/util/u_blit.h
@@ -46,6 +46,8 @@ struct pipe_resource;
 struct pipe_sampler_view;
 struct pipe_surface;
 
+#define BLIT_WRITEMASK_Z         1
+#define BLIT_WRITEMASK_STENCIL   2
 
 extern struct blit_state *
 util_create_blit(struct pipe_context *pipe, struct cso_context *cso);
@@ -64,7 +66,7 @@ util_blit_pixels(struct blit_state *ctx,
                  int dstX0, int dstY0,
                  int dstX1, int dstY1,
                  float z, uint filter,
-                 uint writemask);
+                 uint writemask, uint zs_writemask);
 
 extern void
 util_blit_pixels_tex(struct blit_state *ctx,