radeonsi: clean up CP DMA emit code

Unify the clear and copy paths, clean up the definitions.
It looks more like a rework. It's a preparation for GDS support,
which might or might not come.

Reviewed-by: Nicolai Hähnle <[email protected]>

1 files changed, 60 insertions, 84 deletions

diff --git a/src/gallium/drivers/radeonsi/si_cp_dma.c b/src/gallium/drivers/radeonsi/si_cp_dma.c
index 08d3dfe67e9..0e8d1e89cc3 100644
--- a/src/gallium/drivers/radeonsi/si_cp_dma.c
+++ b/src/gallium/drivers/radeonsi/si_cp_dma.c
@@ -28,50 +28,71 @@
 #include "sid.h"
 #include "radeon/r600_cs.h"
 
+/* Alignment for optimal performance. */
+#define CP_DMA_ALIGNMENT	32
+/* The max number of bytes to copy per packet. */
+#define CP_DMA_MAX_BYTE_COUNT	((1 << 21) - CP_DMA_ALIGNMENT)
 
-/* Set this if you want the 3D engine to wait until CP DMA is done.
+/* Set this if you want the ME to wait until CP DMA is done.
  * It should be set on the last CP DMA packet. */
-#define R600_CP_DMA_SYNC	(1 << 0) /* R600+ */
+#define CP_DMA_SYNC		(1 << 0)
 
 /* Set this if the source data was used as a destination in a previous CP DMA
  * packet. It's for preventing a read-after-write (RAW) hazard between two
  * CP DMA packets. */
-#define SI_CP_DMA_RAW_WAIT	(1 << 1) /* SI+ */
-#define CIK_CP_DMA_USE_L2	(1 << 2)
+#define CP_DMA_RAW_WAIT		(1 << 1)
+#define CP_DMA_USE_L2		(1 << 2) /* CIK+ */
+#define CP_DMA_CLEAR		(1 << 3)
 
-/* Emit a CP DMA packet to do a copy from one buffer to another.
- * The size must fit in bits [20:0].
+/* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
+ * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
+ * clear value.
  */
-static void si_emit_cp_dma_copy_buffer(struct si_context *sctx,
-				       uint64_t dst_va, uint64_t src_va,
-				       unsigned size, unsigned flags)
+static void si_emit_cp_dma(struct si_context *sctx, uint64_t dst_va,
+			   uint64_t src_va, unsigned size, unsigned flags,
+			   enum r600_coherency coher)
 {
 	struct radeon_winsys_cs *cs = sctx->b.gfx.cs;
-	uint32_t sync_flag = flags & R600_CP_DMA_SYNC ? S_411_CP_SYNC(1) : 0;
-	uint32_t wr_confirm = !(flags & R600_CP_DMA_SYNC) ? S_414_DISABLE_WR_CONFIRM(1) : 0;
-	uint32_t raw_wait = flags & SI_CP_DMA_RAW_WAIT ? S_414_RAW_WAIT(1) : 0;
-	uint32_t sel = flags & CIK_CP_DMA_USE_L2 ?
-			   S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2) |
-			   S_411_DSL_SEL(V_411_DST_ADDR_TC_L2) : 0;
+	uint32_t header = 0, command = S_414_BYTE_COUNT(size);
 
 	assert(size);
-	assert((size & ((1<<21)-1)) == size);
+	assert(size <= CP_DMA_MAX_BYTE_COUNT);
+
+	/* Sync flags. */
+	if (flags & CP_DMA_SYNC)
+		header |= S_411_CP_SYNC(1);
+	else
+		command |= S_414_DISABLE_WR_CONFIRM(1);
+
+	if (flags & CP_DMA_RAW_WAIT)
+		command |= S_414_RAW_WAIT(1);
+
+	/* Src and dst flags. */
+	if (flags & CP_DMA_USE_L2)
+		header |= S_411_DSL_SEL(V_411_DST_ADDR_TC_L2);
+
+	if (flags & CP_DMA_CLEAR)
+		header |= S_411_SRC_SEL(V_411_DATA);
+	else if (flags & CP_DMA_USE_L2)
+		header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2);
 
 	if (sctx->b.chip_class >= CIK) {
 		radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, 0));
-		radeon_emit(cs, sync_flag | sel);	/* CP_SYNC [31] */
-		radeon_emit(cs, src_va);		/* SRC_ADDR_LO [31:0] */
-		radeon_emit(cs, src_va >> 32);		/* SRC_ADDR_HI [31:0] */
-		radeon_emit(cs, dst_va);		/* DST_ADDR_LO [31:0] */
-		radeon_emit(cs, dst_va >> 32);		/* DST_ADDR_HI [31:0] */
-		radeon_emit(cs, size | wr_confirm | raw_wait);	/* COMMAND [29:22] | BYTE_COUNT [20:0] */
+		radeon_emit(cs, header);
+		radeon_emit(cs, src_va);	/* SRC_ADDR_LO [31:0] */
+		radeon_emit(cs, src_va >> 32);	/* SRC_ADDR_HI [31:0] */
+		radeon_emit(cs, dst_va);	/* DST_ADDR_LO [31:0] */
+		radeon_emit(cs, dst_va >> 32);	/* DST_ADDR_HI [31:0] */
+		radeon_emit(cs, command);
 	} else {
+		header |= S_411_SRC_ADDR_HI(src_va >> 32);
+
 		radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, 0));
-		radeon_emit(cs, src_va);			/* SRC_ADDR_LO [31:0] */
-		radeon_emit(cs, sync_flag | ((src_va >> 32) & 0xffff)); /* CP_SYNC [31] | SRC_ADDR_HI [15:0] */
-		radeon_emit(cs, dst_va);			/* DST_ADDR_LO [31:0] */
-		radeon_emit(cs, (dst_va >> 32) & 0xffff);	/* DST_ADDR_HI [15:0] */
-		radeon_emit(cs, size | wr_confirm | raw_wait);	/* COMMAND [29:22] | BYTE_COUNT [20:0] */
+		radeon_emit(cs, src_va);	/* SRC_ADDR_LO [31:0] */
+		radeon_emit(cs, header);	/* SRC_ADDR_HI [15:0] + flags. */
+		radeon_emit(cs, dst_va);	/* DST_ADDR_LO [31:0] */
+		radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
+		radeon_emit(cs, command);
 	}
 
 	/* CP DMA is executed in ME, but index buffers are read by PFP.
@@ -79,46 +100,7 @@ static void si_emit_cp_dma_copy_buffer(struct si_context *sctx,
 	 * indices. If we wanted to execute CP DMA in PFP, this packet
 	 * should precede it.
 	 */
-	if (sync_flag) {
-		radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
-		radeon_emit(cs, 0);
-	}
-}
-
-/* Emit a CP DMA packet to clear a buffer. The size must fit in bits [20:0]. */
-static void si_emit_cp_dma_clear_buffer(struct si_context *sctx,
-					uint64_t dst_va, unsigned size,
-					uint32_t clear_value, unsigned flags,
-					enum r600_coherency coher)
-{
-	struct radeon_winsys_cs *cs = sctx->b.gfx.cs;
-	uint32_t sync_flag = flags & R600_CP_DMA_SYNC ? S_411_CP_SYNC(1) : 0;
-	uint32_t wr_confirm = !(flags & R600_CP_DMA_SYNC) ? S_414_DISABLE_WR_CONFIRM(1) : 0;
-	uint32_t raw_wait = flags & SI_CP_DMA_RAW_WAIT ? S_414_RAW_WAIT(1) : 0;
-	uint32_t dst_sel = flags & CIK_CP_DMA_USE_L2 ? S_411_DSL_SEL(V_411_DST_ADDR_TC_L2) : 0;
-
-	assert(size);
-	assert((size & ((1<<21)-1)) == size);
-
-	if (sctx->b.chip_class >= CIK) {
-		radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, 0));
-		radeon_emit(cs, sync_flag | dst_sel | S_411_SRC_SEL(V_411_DATA)); /* CP_SYNC [31] | SRC_SEL[30:29] */
-		radeon_emit(cs, clear_value);		/* DATA [31:0] */
-		radeon_emit(cs, 0);
-		radeon_emit(cs, dst_va);		/* DST_ADDR_LO [31:0] */
-		radeon_emit(cs, dst_va >> 32);		/* DST_ADDR_HI [15:0] */
-		radeon_emit(cs, size | wr_confirm | raw_wait);	/* COMMAND [29:22] | BYTE_COUNT [20:0] */
-	} else {
-		radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, 0));
-		radeon_emit(cs, clear_value);		/* DATA [31:0] */
-		radeon_emit(cs, sync_flag | S_411_SRC_SEL(V_411_DATA)); /* CP_SYNC [31] | SRC_SEL[30:29] */
-		radeon_emit(cs, dst_va);			/* DST_ADDR_LO [31:0] */
-		radeon_emit(cs, (dst_va >> 32) & 0xffff);	/* DST_ADDR_HI [15:0] */
-		radeon_emit(cs, size | wr_confirm | raw_wait);	/* COMMAND [29:22] | BYTE_COUNT [20:0] */
-	}
-
-	/* See "copy_buffer" for explanation. */
-	if (coher == R600_COHERENCY_SHADER && sync_flag) {
+	if (coher == R600_COHERENCY_SHADER && flags & CP_DMA_SYNC) {
 		radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
 		radeon_emit(cs, 0);
 	}
@@ -143,7 +125,7 @@ static unsigned get_flush_flags(struct si_context *sctx, enum r600_coherency coh
 static unsigned get_tc_l2_flag(struct si_context *sctx, enum r600_coherency coher)
 {
 	return coher == R600_COHERENCY_SHADER &&
-	       sctx->b.chip_class >= CIK ? CIK_CP_DMA_USE_L2 : 0;
+	       sctx->b.chip_class >= CIK ? CP_DMA_USE_L2 : 0;
 }
 
 static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst,
@@ -171,21 +153,16 @@ static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst
 	 */
 	if (sctx->b.flags) {
 		si_emit_cache_flush(sctx);
-		*flags |= SI_CP_DMA_RAW_WAIT;
+		*flags |= CP_DMA_RAW_WAIT;
 	}
 
 	/* Do the synchronization after the last dma, so that all data
 	 * is written to memory.
 	 */
 	if (byte_count == remaining_size)
-		*flags |= R600_CP_DMA_SYNC;
+		*flags |= CP_DMA_SYNC;
 }
 
-/* Alignment for optimal performance. */
-#define CP_DMA_ALIGNMENT	32
-/* The max number of bytes to copy per packet. */
-#define CP_DMA_MAX_BYTE_COUNT	((1 << 21) - CP_DMA_ALIGNMENT)
-
 static void si_clear_buffer(struct pipe_context *ctx, struct pipe_resource *dst,
 			    uint64_t offset, uint64_t size, unsigned value,
 			    enum r600_coherency coher)
@@ -224,13 +201,12 @@ static void si_clear_buffer(struct pipe_context *ctx, struct pipe_resource *dst,
 
 	while (size) {
 		unsigned byte_count = MIN2(size, CP_DMA_MAX_BYTE_COUNT);
-		unsigned dma_flags = tc_l2_flag;
+		unsigned dma_flags = tc_l2_flag  | CP_DMA_CLEAR;
 
 		si_cp_dma_prepare(sctx, dst, NULL, byte_count, size, &dma_flags);
 
 		/* Emit the clear packet. */
-		si_emit_cp_dma_clear_buffer(sctx, va, byte_count, value,
-					    dma_flags, coher);
+		si_emit_cp_dma(sctx, va, value, byte_count, dma_flags, coher);
 
 		size -= byte_count;
 		va += byte_count;
@@ -273,8 +249,8 @@ static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size)
 			  &sctx->scratch_buffer->b.b, size, size, &dma_flags);
 
 	va = sctx->scratch_buffer->gpu_address;
-	si_emit_cp_dma_copy_buffer(sctx, va, va + CP_DMA_ALIGNMENT, size,
-				   dma_flags);
+	si_emit_cp_dma(sctx, va, va + CP_DMA_ALIGNMENT, size, dma_flags,
+		       R600_COHERENCY_SHADER);
 }
 
 void si_copy_buffer(struct si_context *sctx,
@@ -337,8 +313,8 @@ void si_copy_buffer(struct si_context *sctx,
 				  size + skipped_size + realign_size,
 				  &dma_flags);
 
-		si_emit_cp_dma_copy_buffer(sctx, main_dst_offset, main_src_offset,
-					   byte_count, dma_flags);
+		si_emit_cp_dma(sctx, main_dst_offset, main_src_offset,
+			       byte_count, dma_flags, R600_COHERENCY_SHADER);
 
 		size -= byte_count;
 		main_src_offset += byte_count;
@@ -353,8 +329,8 @@ void si_copy_buffer(struct si_context *sctx,
 				  skipped_size + realign_size,
 				  &dma_flags);
 
-		si_emit_cp_dma_copy_buffer(sctx, dst_offset, src_offset,
-					   skipped_size, dma_flags);
+		si_emit_cp_dma(sctx, dst_offset, src_offset, skipped_size,
+			       dma_flags, R600_COHERENCY_SHADER);
 	}
 
 	/* Finally, realign the engine if the size wasn't aligned. */