radeonsi: make si_gfx_write_event_eop more configurable

6 files changed, 34 insertions, 15 deletions

diff --git a/src/amd/common/sid.h b/src/amd/common/sid.h
index b3321ea3a77..3e36eb2d046 100644
--- a/src/amd/common/sid.h
+++ b/src/amd/common/sid.h
@@ -181,6 +181,9 @@
 #define PKT3_COND_WRITE                        0x45
 #define PKT3_EVENT_WRITE                       0x46
 #define PKT3_EVENT_WRITE_EOP                   0x47 /* not on GFX9 */
+#define         EOP_DST_SEL(x)				((x) << 16)
+#define			EOP_DST_SEL_MEM			0
+#define			EOP_DST_SEL_TC_L2		1
 #define         EOP_INT_SEL(x)                          ((x) << 24)
 #define			EOP_INT_SEL_NONE			0
 #define			EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM	3
@@ -189,6 +192,8 @@
 #define			EOP_DATA_SEL_VALUE_32BIT	1
 #define			EOP_DATA_SEL_VALUE_64BIT	2
 #define			EOP_DATA_SEL_TIMESTAMP		3
+#define			EOP_DATA_SEL_GDS		5
+#define		EOP_DATA_GDS(dw_offset, num_dwords)	((dw_offset) | ((unsigned)(num_dwords) << 16))
 /* CP DMA bug: Any use of CP_DMA.DST_SEL=TC must be avoided when EOS packets
  * are used. Use DST_SEL=MC instead. For prefetch, use SRC_SEL=TC and
  * DST_SEL=MC. Only CIK chips are affected.
diff --git a/src/gallium/drivers/radeonsi/si_fence.c b/src/gallium/drivers/radeonsi/si_fence.c
index abb7057f299..f1e0fac230b 100644
--- a/src/gallium/drivers/radeonsi/si_fence.c
+++ b/src/gallium/drivers/radeonsi/si_fence.c
@@ -58,7 +58,9 @@ struct si_multi_fence {
  *
  * \param event		EVENT_TYPE_*
  * \param event_flags	Optional cache flush flags (TC)
- * \param data_sel	1 = fence, 3 = timestamp
+ * \param dst_sel       MEM or TC_L2
+ * \param int_sel       NONE or SEND_DATA_AFTER_WR_CONFIRM
+ * \param data_sel	DISCARD, VALUE_32BIT, TIMESTAMP, or GDS
  * \param buf		Buffer
  * \param va		GPU address
  * \param old_value	Previous fence value (for a bug workaround)
@@ -66,20 +68,18 @@ struct si_multi_fence {
  */
 void si_gfx_write_event_eop(struct si_context *ctx,
 			    unsigned event, unsigned event_flags,
-			    unsigned data_sel,
+			    unsigned dst_sel, unsigned int_sel, unsigned data_sel,
 			    struct r600_resource *buf, uint64_t va,
 			    uint32_t new_fence, unsigned query_type)
 {
 	struct radeon_cmdbuf *cs = ctx->gfx_cs;
 	unsigned op = EVENT_TYPE(event) |
-		      EVENT_INDEX(5) |
+		      EVENT_INDEX(event == V_028A90_CS_DONE ||
+				  event == V_028A90_PS_DONE ? 6 : 5) |
 		      event_flags;
-	unsigned sel = EOP_DATA_SEL(data_sel);
-
-	/* Wait for write confirmation before writing data, but don't send
-	 * an interrupt. */
-	if (data_sel != EOP_DATA_SEL_DISCARD)
-		sel |= EOP_INT_SEL(EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM);
+	unsigned sel = EOP_DST_SEL(dst_sel) |
+		       EOP_INT_SEL(int_sel) |
+		       EOP_DATA_SEL(data_sel);
 
 	if (ctx->chip_class >= GFX9) {
 		/* A ZPASS_DONE or PIXEL_STAT_DUMP_EVENT (of the DB occlusion
@@ -275,7 +275,10 @@ static void si_fine_fence_set(struct si_context *ctx,
 		radeon_emit(cs, fence_va >> 32);
 		radeon_emit(cs, 0x80000000);
 	} else if (flags & PIPE_FLUSH_BOTTOM_OF_PIPE) {
-		si_gfx_write_event_eop(ctx, V_028A90_BOTTOM_OF_PIPE_TS, 0,
+		si_gfx_write_event_eop(ctx,
+				       V_028A90_BOTTOM_OF_PIPE_TS, 0,
+				       EOP_DST_SEL_MEM,
+				       EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM,
 				       EOP_DATA_SEL_VALUE_32BIT,
 				       NULL, fence_va, 0x80000000,
 				       PIPE_QUERY_GPU_FINISHED);
diff --git a/src/gallium/drivers/radeonsi/si_perfcounter.c b/src/gallium/drivers/radeonsi/si_perfcounter.c
index de71572c8aa..f3ef3d28c8a 100644
--- a/src/gallium/drivers/radeonsi/si_perfcounter.c
+++ b/src/gallium/drivers/radeonsi/si_perfcounter.c
@@ -581,6 +581,8 @@ static void si_pc_emit_stop(struct si_context *sctx,
 	struct radeon_cmdbuf *cs = sctx->gfx_cs;
 
 	si_gfx_write_event_eop(sctx, V_028A90_BOTTOM_OF_PIPE_TS, 0,
+			       EOP_DST_SEL_MEM,
+			       EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM,
 			       EOP_DATA_SEL_VALUE_32BIT,
 			       buffer, va, 0, SI_NOT_QUERY);
 	si_gfx_wait_fence(sctx, va, 0, 0xffffffff);
diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h
index 29d7e555a0c..73c54df3a03 100644
--- a/src/gallium/drivers/radeonsi/si_pipe.h
+++ b/src/gallium/drivers/radeonsi/si_pipe.h
@@ -1172,7 +1172,7 @@ void si_screen_clear_buffer(struct si_screen *sscreen, struct pipe_resource *dst
 /* si_fence.c */
 void si_gfx_write_event_eop(struct si_context *ctx,
 			    unsigned event, unsigned event_flags,
-			    unsigned data_sel,
+			    unsigned dst_sel, unsigned int_sel, unsigned data_sel,
 			    struct r600_resource *buf, uint64_t va,
 			    uint32_t new_fence, unsigned query_type);
 unsigned si_gfx_write_fence_dwords(struct si_screen *screen);
diff --git a/src/gallium/drivers/radeonsi/si_query.c b/src/gallium/drivers/radeonsi/si_query.c
index bdd7e2c060c..45c8e146ecf 100644
--- a/src/gallium/drivers/radeonsi/si_query.c
+++ b/src/gallium/drivers/radeonsi/si_query.c
@@ -891,7 +891,9 @@ static void si_query_hw_do_emit_stop(struct si_context *sctx,
 		/* fall through */
 	case PIPE_QUERY_TIMESTAMP:
 		si_gfx_write_event_eop(sctx, V_028A90_BOTTOM_OF_PIPE_TS,
-				       0, EOP_DATA_SEL_TIMESTAMP, NULL, va,
+				       0, EOP_DST_SEL_MEM,
+				       EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM,
+				       EOP_DATA_SEL_TIMESTAMP, NULL, va,
 				       0, query->b.type);
 		fence_va = va + 8;
 		break;
@@ -913,11 +915,14 @@ static void si_query_hw_do_emit_stop(struct si_context *sctx,
 	radeon_add_to_buffer_list(sctx, sctx->gfx_cs, query->buffer.buf, RADEON_USAGE_WRITE,
 				  RADEON_PRIO_QUERY);
 
-	if (fence_va)
+	if (fence_va) {
 		si_gfx_write_event_eop(sctx, V_028A90_BOTTOM_OF_PIPE_TS, 0,
+				       EOP_DST_SEL_MEM,
+				       EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM,
 				       EOP_DATA_SEL_VALUE_32BIT,
 				       query->buffer.buf, fence_va, 0x80000000,
 				       query->b.type);
+	}
 }
 
 static void si_query_hw_emit_stop(struct si_context *sctx,
diff --git a/src/gallium/drivers/radeonsi/si_state_draw.c b/src/gallium/drivers/radeonsi/si_state_draw.c
index c9fb97d27e3..81eb34d75e2 100644
--- a/src/gallium/drivers/radeonsi/si_state_draw.c
+++ b/src/gallium/drivers/radeonsi/si_state_draw.c
@@ -918,8 +918,10 @@ void si_emit_cache_flush(struct si_context *sctx)
 
 			/* Necessary for DCC */
 			if (sctx->chip_class == VI)
-				si_gfx_write_event_eop(sctx, V_028A90_FLUSH_AND_INV_CB_DATA_TS,
-						       0, EOP_DATA_SEL_DISCARD, NULL,
+				si_gfx_write_event_eop(sctx,
+						       V_028A90_FLUSH_AND_INV_CB_DATA_TS,
+						       0, EOP_DST_SEL_MEM, EOP_INT_SEL_NONE,
+						       EOP_DATA_SEL_DISCARD, NULL,
 						       0, 0, SI_NOT_QUERY);
 		}
 		if (flags & SI_CONTEXT_FLUSH_AND_INV_DB)
@@ -1034,6 +1036,8 @@ void si_emit_cache_flush(struct si_context *sctx)
 		sctx->wait_mem_number++;
 
 		si_gfx_write_event_eop(sctx, cb_db_event, tc_flags,
+				       EOP_DST_SEL_MEM,
+				       EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM,
 				       EOP_DATA_SEL_VALUE_32BIT,
 				       sctx->wait_mem_scratch, va,
 				       sctx->wait_mem_number, SI_NOT_QUERY);