freedreno/drm: remove idx_lock

Since it ends up contended, it is a bit of a bottleneck for workloads
with high driver overhead.  Worth nearly +10% at gfxbench driver2.

Signed-off-by: Rob Clark <[email protected]>

3 files changed, 24 insertions, 29 deletions

diff --git a/src/freedreno/drm/msm_priv.h b/src/freedreno/drm/msm_priv.h
index acf22bcb6f8..fd4267fa371 100644
--- a/src/freedreno/drm/msm_priv.h
+++ b/src/freedreno/drm/msm_priv.h
@@ -66,11 +66,6 @@ struct fd_submit * msm_submit_sp_new(struct fd_pipe *pipe);
 struct msm_bo {
 	struct fd_bo base;
 	uint64_t offset;
-	/* to avoid excess hashtable lookups, cache the ring this bo was
-	 * last emitted on (since that will probably also be the next ring
-	 * it is emitted on)
-	 */
-	unsigned current_submit_seqno;
 	uint32_t idx;
 };
 FD_DEFINE_CAST(fd_bo, msm_bo);
@@ -138,4 +133,6 @@ grow(void *ptr, uint16_t nr, uint16_t *max, uint16_t sz)
 	(x)->nr_ ## name ++; \
 })
 
+#define READ_ONCE(x) (*(volatile __typeof__(x) *)&(x))
+
 #endif /* MSM_PRIV_H_ */
diff --git a/src/freedreno/drm/msm_ringbuffer.c b/src/freedreno/drm/msm_ringbuffer.c
index 369f26f9837..c4b352ee283 100644
--- a/src/freedreno/drm/msm_ringbuffer.c
+++ b/src/freedreno/drm/msm_ringbuffer.c
@@ -41,8 +41,6 @@
 
 #define INIT_SIZE 0x1000
 
-static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER;
-
 
 struct msm_submit {
 	struct fd_submit base;
@@ -50,8 +48,6 @@ struct msm_submit {
 	DECLARE_ARRAY(struct drm_msm_gem_submit_bo, submit_bos);
 	DECLARE_ARRAY(struct fd_bo *, bos);
 
-	unsigned seqno;
-
 	/* maps fd_bo to idx in bos table: */
 	struct hash_table *bo_table;
 
@@ -146,10 +142,15 @@ append_bo(struct msm_submit *submit, struct fd_bo *bo, uint32_t flags)
 {
 	struct msm_bo *msm_bo = to_msm_bo(bo);
 	uint32_t idx;
-	pthread_mutex_lock(&idx_lock);
-	if (likely(msm_bo->current_submit_seqno == submit->seqno)) {
-		idx = msm_bo->idx;
-	} else {
+
+	/* NOTE: it is legal to use the same bo on different threads for
+	 * different submits.  But it is not legal to use the same submit
+	 * from given threads.
+	 */
+	idx = READ_ONCE(msm_bo->idx);
+
+	if (unlikely((idx >= submit->nr_submit_bos) ||
+			(submit->submit_bos[idx].handle != bo->handle))) {
 		uint32_t hash = _mesa_hash_pointer(bo);
 		struct hash_entry *entry;
 
@@ -170,14 +171,14 @@ append_bo(struct msm_submit *submit, struct fd_bo *bo, uint32_t flags)
 			_mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo,
 					(void *)(uintptr_t)idx);
 		}
-		msm_bo->current_submit_seqno = submit->seqno;
 		msm_bo->idx = idx;
 	}
-	pthread_mutex_unlock(&idx_lock);
+
 	if (flags & FD_RELOC_READ)
 		submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_READ;
 	if (flags & FD_RELOC_WRITE)
 		submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_WRITE;
+
 	return idx;
 }
 
@@ -455,9 +456,7 @@ msm_submit_new(struct fd_pipe *pipe)
 {
 	struct msm_submit *msm_submit = calloc(1, sizeof(*msm_submit));
 	struct fd_submit *submit;
-	static unsigned submit_cnt = 0;
 
-	msm_submit->seqno = ++submit_cnt;
 	msm_submit->bo_table = _mesa_hash_table_create(NULL,
 			_mesa_hash_pointer, _mesa_key_pointer_equal);
 	msm_submit->ring_set = _mesa_set_create(NULL,
diff --git a/src/freedreno/drm/msm_ringbuffer_sp.c b/src/freedreno/drm/msm_ringbuffer_sp.c
index 2b8f531721e..c87ad512810 100644
--- a/src/freedreno/drm/msm_ringbuffer_sp.c
+++ b/src/freedreno/drm/msm_ringbuffer_sp.c
@@ -41,8 +41,6 @@
 
 #define INIT_SIZE 0x1000
 
-static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER;
-
 
 struct msm_submit_sp {
 	struct fd_submit base;
@@ -50,8 +48,6 @@ struct msm_submit_sp {
 	DECLARE_ARRAY(struct drm_msm_gem_submit_bo, submit_bos);
 	DECLARE_ARRAY(struct fd_bo *, bos);
 
-	unsigned seqno;
-
 	/* maps fd_bo to idx in bos table: */
 	struct hash_table *bo_table;
 
@@ -124,10 +120,15 @@ append_bo(struct msm_submit_sp *submit, struct fd_bo *bo, uint32_t flags)
 {
 	struct msm_bo *msm_bo = to_msm_bo(bo);
 	uint32_t idx;
-	pthread_mutex_lock(&idx_lock);
-	if (likely(msm_bo->current_submit_seqno == submit->seqno)) {
-		idx = msm_bo->idx;
-	} else {
+
+	/* NOTE: it is legal to use the same bo on different threads for
+	 * different submits.  But it is not legal to use the same submit
+	 * from given threads.
+	 */
+	idx = READ_ONCE(msm_bo->idx);
+
+	if (unlikely((idx >= submit->nr_submit_bos) ||
+			(submit->submit_bos[idx].handle != bo->handle))) {
 		uint32_t hash = _mesa_hash_pointer(bo);
 		struct hash_entry *entry;
 
@@ -148,16 +149,16 @@ append_bo(struct msm_submit_sp *submit, struct fd_bo *bo, uint32_t flags)
 			_mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo,
 					(void *)(uintptr_t)idx);
 		}
-		msm_bo->current_submit_seqno = submit->seqno;
 		msm_bo->idx = idx;
 	}
-	pthread_mutex_unlock(&idx_lock);
+
 	if (flags & FD_RELOC_READ)
 		submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_READ;
 	if (flags & FD_RELOC_WRITE)
 		submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_WRITE;
 	if (flags & FD_RELOC_DUMP)
 		submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_DUMP;
+
 	return idx;
 }
 
@@ -337,9 +338,7 @@ msm_submit_sp_new(struct fd_pipe *pipe)
 {
 	struct msm_submit_sp *msm_submit = calloc(1, sizeof(*msm_submit));
 	struct fd_submit *submit;
-	static unsigned submit_cnt = 0;
 
-	msm_submit->seqno = ++submit_cnt;
 	msm_submit->bo_table = _mesa_hash_table_create(NULL,
 			_mesa_hash_pointer, _mesa_key_pointer_equal);
 	// TODO tune size: