radeonsi/gfx9: add support for PIPE_ALIGNED=0

Needed by displayable DCC.

We need to flush L2 after rendering if PIPE_ALIGNED=0 and DCC is enabled.

4 files changed, 30 insertions, 12 deletions

diff --git a/src/gallium/drivers/radeonsi/si_blit.c b/src/gallium/drivers/radeonsi/si_blit.c
index f39cb5d143f..7613a63e3cb 100644
--- a/src/gallium/drivers/radeonsi/si_blit.c
+++ b/src/gallium/drivers/radeonsi/si_blit.c
@@ -421,7 +421,7 @@ si_decompress_depth(struct si_context *sctx,
 	 */
 	if (copy_planes && tex->buffer.b.b.nr_samples > 1)
 		si_make_CB_shader_coherent(sctx, tex->buffer.b.b.nr_samples,
-					   false);
+					   false, true /* no DCC */);
 }
 
 static void
@@ -534,7 +534,8 @@ static void si_blit_decompress_color(struct si_context *sctx,
 
 	sctx->decompression_enabled = false;
 	si_make_CB_shader_coherent(sctx, tex->buffer.b.b.nr_samples,
-				   vi_dcc_enabled(tex, first_level));
+				   vi_dcc_enabled(tex, first_level),
+				   tex->surface.u.gfx9.dcc.pipe_aligned);
 }
 
 static void
@@ -1076,7 +1077,7 @@ static void si_do_CB_resolve(struct si_context *sctx,
 	si_blitter_end(sctx);
 
 	/* Flush caches for possible texturing. */
-	si_make_CB_shader_coherent(sctx, 1, false);
+	si_make_CB_shader_coherent(sctx, 1, false, true /* no DCC */);
 }
 
 static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
diff --git a/src/gallium/drivers/radeonsi/si_compute_blit.c b/src/gallium/drivers/radeonsi/si_compute_blit.c
index a7453099ac6..61bef999357 100644
--- a/src/gallium/drivers/radeonsi/si_compute_blit.c
+++ b/src/gallium/drivers/radeonsi/si_compute_blit.c
@@ -324,7 +324,11 @@ void si_compute_copy_image(struct si_context *sctx,
 	si_compute_internal_begin(sctx);
 	sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH |
 		       si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM);
-	si_make_CB_shader_coherent(sctx, dst->nr_samples, true);
+
+	/* src and dst have the same number of samples. */
+	si_make_CB_shader_coherent(sctx, src->nr_samples, true,
+				   /* Only src can have DCC.*/
+				   ((struct si_texture*)src)->surface.u.gfx9.dcc.pipe_aligned);
 
 	struct pipe_constant_buffer saved_cb = {};
 	si_get_pipe_constant_buffer(sctx, PIPE_SHADER_COMPUTE, 0, &saved_cb);
@@ -447,7 +451,8 @@ void si_compute_clear_render_target(struct pipe_context *ctx,
 
 	sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH |
 		       si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM);
-	si_make_CB_shader_coherent(sctx, dstsurf->texture->nr_samples, true);
+	si_make_CB_shader_coherent(sctx, dstsurf->texture->nr_samples, true,
+				   true /* DCC is not possible with image stores */);
 
 	struct pipe_constant_buffer saved_cb = {};
 	si_get_pipe_constant_buffer(sctx, PIPE_SHADER_COMPUTE, 0, &saved_cb);
diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h
index b6858b46ec0..16bd074f998 100644
--- a/src/gallium/drivers/radeonsi/si_pipe.h
+++ b/src/gallium/drivers/radeonsi/si_pipe.h
@@ -638,6 +638,7 @@ struct si_framebuffer {
 	bool				any_dst_linear;
 	bool				CB_has_shader_readable_metadata;
 	bool				DB_has_shader_readable_metadata;
+	bool				all_DCC_pipe_aligned;
 };
 
 enum si_quant_mode {
@@ -1524,7 +1525,7 @@ si_saved_cs_reference(struct si_saved_cs **dst, struct si_saved_cs *src)
 
 static inline void
 si_make_CB_shader_coherent(struct si_context *sctx, unsigned num_samples,
-			   bool shaders_read_metadata)
+			   bool shaders_read_metadata, bool dcc_pipe_aligned)
 {
 	sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_CB |
 		       SI_CONTEXT_INV_VMEM_L1;
@@ -1534,7 +1535,8 @@ si_make_CB_shader_coherent(struct si_context *sctx, unsigned num_samples,
 		 * L2 metadata must be flushed if shaders read metadata.
 		 * (DCC, CMASK).
 		 */
-		if (num_samples >= 2)
+		if (num_samples >= 2 ||
+		    (shaders_read_metadata && !dcc_pipe_aligned))
 			sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
 		else if (shaders_read_metadata)
 			sctx->flags |= SI_CONTEXT_INV_L2_METADATA;
diff --git a/src/gallium/drivers/radeonsi/si_state.c b/src/gallium/drivers/radeonsi/si_state.c
index 3c29b4c92ed..e39387a6080 100644
--- a/src/gallium/drivers/radeonsi/si_state.c
+++ b/src/gallium/drivers/radeonsi/si_state.c
@@ -2807,9 +2807,11 @@ static void si_set_framebuffer_state(struct pipe_context *ctx,
 	 *
 	 * Only flush and wait for CB if there is actually a bound color buffer.
 	 */
-	if (sctx->framebuffer.uncompressed_cb_mask)
+	if (sctx->framebuffer.uncompressed_cb_mask) {
 		si_make_CB_shader_coherent(sctx, sctx->framebuffer.nr_samples,
-					   sctx->framebuffer.CB_has_shader_readable_metadata);
+					   sctx->framebuffer.CB_has_shader_readable_metadata,
+					   sctx->framebuffer.all_DCC_pipe_aligned);
+	}
 
 	sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH;
 
@@ -2858,6 +2860,7 @@ static void si_set_framebuffer_state(struct pipe_context *ctx,
 	sctx->framebuffer.any_dst_linear = false;
 	sctx->framebuffer.CB_has_shader_readable_metadata = false;
 	sctx->framebuffer.DB_has_shader_readable_metadata = false;
+	sctx->framebuffer.all_DCC_pipe_aligned = true;
 	unsigned num_bpp64_colorbufs = 0;
 
 	for (i = 0; i < state->nr_cbufs; i++) {
@@ -2908,9 +2911,14 @@ static void si_set_framebuffer_state(struct pipe_context *ctx,
 		if (tex->surface.bpe >= 8)
 			num_bpp64_colorbufs++;
 
-		if (vi_dcc_enabled(tex, surf->base.u.tex.level))
+		if (vi_dcc_enabled(tex, surf->base.u.tex.level)) {
 			sctx->framebuffer.CB_has_shader_readable_metadata = true;
 
+			if (sctx->chip_class >= GFX9 &&
+			    !tex->surface.u.gfx9.dcc.pipe_aligned)
+				sctx->framebuffer.all_DCC_pipe_aligned = false;
+		}
+
 		si_context_add_resource_size(sctx, surf->base.texture);
 
 		p_atomic_inc(&tex->framebuffers_bound);
@@ -4700,9 +4708,11 @@ static void si_texture_barrier(struct pipe_context *ctx, unsigned flags)
 	si_update_fb_dirtiness_after_rendering(sctx);
 
 	/* Multisample surfaces are flushed in si_decompress_textures. */
-	if (sctx->framebuffer.uncompressed_cb_mask)
+	if (sctx->framebuffer.uncompressed_cb_mask) {
 		si_make_CB_shader_coherent(sctx, sctx->framebuffer.nr_samples,
-					   sctx->framebuffer.CB_has_shader_readable_metadata);
+					   sctx->framebuffer.CB_has_shader_readable_metadata,
+					   sctx->framebuffer.all_DCC_pipe_aligned);
+	}
 }
 
 /* This only ensures coherency for shader image/buffer stores. */