1 files changed, 168 insertions, 50 deletions

diff --git a/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c b/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c
index c4b2232ce9e..0cd870b7c89 100644
--- a/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c
+++ b/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c
@@ -68,6 +68,10 @@ struct radv_amdgpu_cs {
 	struct radeon_winsys_bo     **virtual_buffers;
 	uint8_t                     *virtual_buffer_priorities;
 	int                         *virtual_buffer_hash_table;
+
+	/* For chips that don't support chaining. */
+	struct radeon_winsys_cs     *old_cs_buffers;
+	unsigned                    num_old_cs_buffers;
 };
 
 static inline struct radv_amdgpu_cs *
@@ -201,6 +205,12 @@ static void radv_amdgpu_cs_destroy(struct radeon_winsys_cs *rcs)
 	for (unsigned i = 0; i < cs->num_old_ib_buffers; ++i)
 		cs->ws->base.buffer_destroy(cs->old_ib_buffers[i]);
 
+	for (unsigned i = 0; i < cs->num_old_cs_buffers; ++i) {
+		struct radeon_winsys_cs *rcs = &cs->old_cs_buffers[i];
+		free(rcs->buf);
+	}
+
+	free(cs->old_cs_buffers);
 	free(cs->old_ib_buffers);
 	free(cs->virtual_buffers);
 	free(cs->virtual_buffer_priorities);
@@ -286,9 +296,46 @@ static void radv_amdgpu_cs_grow(struct radeon_winsys_cs *_cs, size_t min_size)
 		/* The total ib size cannot exceed limit_dws dwords. */
 		if (ib_dws > limit_dws)
 		{
-			cs->failed = true;
+			/* The maximum size in dwords has been reached,
+			 * try to allocate a new one.
+			 */
+			if (cs->num_old_cs_buffers + 1 >= AMDGPU_CS_MAX_IBS_PER_SUBMIT) {
+				/* TODO: Allow to submit more than 4 IBs. */
+				fprintf(stderr, "amdgpu: Maximum number of IBs "
+						"per submit reached.\n");
+				cs->failed = true;
+				cs->base.cdw = 0;
+				return;
+			}
+
+			cs->old_cs_buffers =
+				realloc(cs->old_cs_buffers,
+				        (cs->num_old_cs_buffers + 1) * sizeof(*cs->old_cs_buffers));
+			if (!cs->old_cs_buffers) {
+				cs->failed = true;
+				cs->base.cdw = 0;
+				return;
+			}
+
+			/* Store the current one for submitting it later. */
+			cs->old_cs_buffers[cs->num_old_cs_buffers].cdw = cs->base.cdw;
+			cs->old_cs_buffers[cs->num_old_cs_buffers].max_dw = cs->base.max_dw;
+			cs->old_cs_buffers[cs->num_old_cs_buffers].buf = cs->base.buf;
+			cs->num_old_cs_buffers++;
+
+			/* Reset the cs, it will be re-allocated below. */
 			cs->base.cdw = 0;
-			return;
+			cs->base.buf = NULL;
+
+			/* Re-compute the number of dwords to allocate. */
+			ib_dws = MAX2(cs->base.cdw + min_size,
+				      MIN2(cs->base.max_dw * 2, limit_dws));
+			if (ib_dws > limit_dws) {
+				fprintf(stderr, "amdgpu: Too high number of "
+						"dwords to allocate\n");
+				cs->failed = true;
+				return;
+			}
 		}
 
 		uint32_t *new_buf = realloc(cs->base.buf, ib_dws * 4);
@@ -400,6 +447,15 @@ static void radv_amdgpu_cs_reset(struct radeon_winsys_cs *_cs)
 		cs->ib.ib_mc_address = radv_amdgpu_winsys_bo(cs->ib_buffer)->base.va;
 		cs->ib_size_ptr = &cs->ib.size;
 		cs->ib.size = 0;
+	} else {
+		for (unsigned i = 0; i < cs->num_old_cs_buffers; ++i) {
+			struct radeon_winsys_cs *rcs = &cs->old_cs_buffers[i];
+			free(rcs->buf);
+		}
+
+		free(cs->old_cs_buffers);
+		cs->old_cs_buffers = NULL;
+		cs->num_old_cs_buffers = 0;
 	}
 }
 
@@ -550,7 +606,8 @@ static void radv_amdgpu_cs_execute_secondary(struct radeon_winsys_cs *_parent,
 static int radv_amdgpu_create_bo_list(struct radv_amdgpu_winsys *ws,
 				      struct radeon_winsys_cs **cs_array,
 				      unsigned count,
-				      struct radv_amdgpu_winsys_bo *extra_bo,
+				      struct radv_amdgpu_winsys_bo **extra_bo_array,
+				      unsigned num_extra_bo,
 				      struct radeon_winsys_cs *extra_cs,
 				      const struct radv_winsys_bo_list *radv_bo_list,
 				      amdgpu_bo_list_handle *bo_list)
@@ -580,7 +637,7 @@ static int radv_amdgpu_create_bo_list(struct radv_amdgpu_winsys *ws,
 					  bo_list);
 		free(handles);
 		pthread_mutex_unlock(&ws->global_bo_list_lock);
-	} else if (count == 1 && !extra_bo && !extra_cs && !radv_bo_list && 
+	} else if (count == 1 && !num_extra_bo && !extra_cs && !radv_bo_list &&
 	           !radv_amdgpu_cs(cs_array[0])->num_virtual_buffers) {
 		struct radv_amdgpu_cs *cs = (struct radv_amdgpu_cs*)cs_array[0];
 		if (cs->num_buffers == 0) {
@@ -590,8 +647,8 @@ static int radv_amdgpu_create_bo_list(struct radv_amdgpu_winsys *ws,
 		r = amdgpu_bo_list_create(ws->dev, cs->num_buffers, cs->handles,
 					  cs->priorities, bo_list);
 	} else {
-		unsigned total_buffer_count = !!extra_bo;
-		unsigned unique_bo_count = !!extra_bo;
+		unsigned total_buffer_count = num_extra_bo;
+		unsigned unique_bo_count = num_extra_bo;
 		for (unsigned i = 0; i < count; ++i) {
 			struct radv_amdgpu_cs *cs = (struct radv_amdgpu_cs*)cs_array[i];
 			total_buffer_count += cs->num_buffers;
@@ -619,9 +676,9 @@ static int radv_amdgpu_create_bo_list(struct radv_amdgpu_winsys *ws,
 			return -ENOMEM;
 		}
 
-		if (extra_bo) {
-			handles[0] = extra_bo->bo;
-			priorities[0] = 8;
+		for (unsigned i = 0; i < num_extra_bo; i++) {
+			handles[i] = extra_bo_array[i]->bo;
+			priorities[i] = 8;
 		}
 
 		for (unsigned i = 0; i < count + !!extra_cs; ++i) {
@@ -773,7 +830,7 @@ static int radv_amdgpu_winsys_cs_submit_chained(struct radeon_winsys_ctx *_ctx,
 		}
 	}
 
-	r = radv_amdgpu_create_bo_list(cs0->ws, cs_array, cs_count, NULL, initial_preamble_cs,
+	r = radv_amdgpu_create_bo_list(cs0->ws, cs_array, cs_count, NULL, 0, initial_preamble_cs,
 	                               radv_bo_list, &bo_list);
 	if (r) {
 		fprintf(stderr, "amdgpu: buffer list creation failed for the "
@@ -842,7 +899,7 @@ static int radv_amdgpu_winsys_cs_submit_fallback(struct radeon_winsys_ctx *_ctx,
 
 		memset(&request, 0, sizeof(request));
 
-		r = radv_amdgpu_create_bo_list(cs0->ws, &cs_array[i], cnt, NULL,
+		r = radv_amdgpu_create_bo_list(cs0->ws, &cs_array[i], cnt, NULL, 0,
 		                               preamble_cs, radv_bo_list, &bo_list);
 		if (r) {
 			fprintf(stderr, "amdgpu: buffer list creation failed "
@@ -923,68 +980,127 @@ static int radv_amdgpu_winsys_cs_submit_sysmem(struct radeon_winsys_ctx *_ctx,
 	assert(cs_count);
 
 	for (unsigned i = 0; i < cs_count;) {
-		struct amdgpu_cs_ib_info ib = {0};
-		struct radeon_winsys_bo *bo = NULL;
+		struct amdgpu_cs_ib_info ibs[AMDGPU_CS_MAX_IBS_PER_SUBMIT] = {0};
+		unsigned number_of_ibs = 1;
+		struct radeon_winsys_bo *bos[AMDGPU_CS_MAX_IBS_PER_SUBMIT] = {0};
 		struct radeon_winsys_cs *preamble_cs = i ? continue_preamble_cs : initial_preamble_cs;
+		struct radv_amdgpu_cs *cs = radv_amdgpu_cs(cs_array[i]);
 		uint32_t *ptr;
 		unsigned cnt = 0;
 		unsigned size = 0;
 		unsigned pad_words = 0;
-		if (preamble_cs)
-			size += preamble_cs->cdw;
 
-		while (i + cnt < cs_count && 0xffff8 - size >= radv_amdgpu_cs(cs_array[i + cnt])->base.cdw) {
-			size += radv_amdgpu_cs(cs_array[i + cnt])->base.cdw;
-			++cnt;
-		}
+		if (cs->num_old_cs_buffers > 0) {
+			/* Special path when the maximum size in dwords has
+			 * been reached because we need to handle more than one
+			 * IB per submit.
+			 */
+			unsigned new_cs_count = cs->num_old_cs_buffers + 1;
+			struct radeon_winsys_cs *new_cs_array[AMDGPU_CS_MAX_IBS_PER_SUBMIT];
+			unsigned idx = 0;
+
+			for (unsigned j = 0; j < cs->num_old_cs_buffers; j++)
+				new_cs_array[idx++] = &cs->old_cs_buffers[j];
+			new_cs_array[idx++] = cs_array[i];
+
+			for (unsigned j = 0; j < new_cs_count; j++) {
+				struct radeon_winsys_cs *rcs = new_cs_array[j];
+				bool needs_preamble = preamble_cs && j == 0;
+				unsigned size = 0;
+
+				if (needs_preamble)
+					size += preamble_cs->cdw;
+				size += rcs->cdw;
+
+				assert(size < 0xffff8);
+
+				while (!size || (size & 7)) {
+					size++;
+					pad_words++;
+				}
 
-		while(!size || (size & 7)) {
-			size++;
-			pad_words++;
-		}
-		assert(cnt);
+				bos[j] = ws->buffer_create(ws, 4 * size, 4096,
+							   RADEON_DOMAIN_GTT,
+							   RADEON_FLAG_CPU_ACCESS |
+							   RADEON_FLAG_NO_INTERPROCESS_SHARING |
+							   RADEON_FLAG_READ_ONLY);
+				ptr = ws->buffer_map(bos[j]);
 
-		bo = ws->buffer_create(ws, 4 * size, 4096, RADEON_DOMAIN_GTT,
-				       RADEON_FLAG_CPU_ACCESS |
-				       RADEON_FLAG_NO_INTERPROCESS_SHARING |
-				       RADEON_FLAG_READ_ONLY);
-		ptr = ws->buffer_map(bo);
+				if (needs_preamble) {
+					memcpy(ptr, preamble_cs->buf, preamble_cs->cdw * 4);
+					ptr += preamble_cs->cdw;
+				}
 
-		if (preamble_cs) {
-			memcpy(ptr, preamble_cs->buf, preamble_cs->cdw * 4);
-			ptr += preamble_cs->cdw;
-		}
+				memcpy(ptr, rcs->buf, 4 * rcs->cdw);
+				ptr += rcs->cdw;
 
-		for (unsigned j = 0; j < cnt; ++j) {
-			struct radv_amdgpu_cs *cs = radv_amdgpu_cs(cs_array[i + j]);
-			memcpy(ptr, cs->base.buf, 4 * cs->base.cdw);
-			ptr += cs->base.cdw;
+				for (unsigned k = 0; k < pad_words; ++k)
+					*ptr++ = pad_word;
 
-		}
+				ibs[j].size = size;
+				ibs[j].ib_mc_address = radv_buffer_get_va(bos[j]);
+			}
 
-		for (unsigned j = 0; j < pad_words; ++j)
-			*ptr++ = pad_word;
+			number_of_ibs = new_cs_count;
+			cnt++;
+		} else {
+			if (preamble_cs)
+				size += preamble_cs->cdw;
 
-		memset(&request, 0, sizeof(request));
+			while (i + cnt < cs_count && 0xffff8 - size >= radv_amdgpu_cs(cs_array[i + cnt])->base.cdw) {
+				size += radv_amdgpu_cs(cs_array[i + cnt])->base.cdw;
+				++cnt;
+			}
+
+			while (!size || (size & 7)) {
+				size++;
+				pad_words++;
+			}
+			assert(cnt);
 
+			bos[0] = ws->buffer_create(ws, 4 * size, 4096,
+						   RADEON_DOMAIN_GTT,
+						   RADEON_FLAG_CPU_ACCESS |
+						   RADEON_FLAG_NO_INTERPROCESS_SHARING |
+						   RADEON_FLAG_READ_ONLY);
+			ptr = ws->buffer_map(bos[0]);
+
+			if (preamble_cs) {
+				memcpy(ptr, preamble_cs->buf, preamble_cs->cdw * 4);
+				ptr += preamble_cs->cdw;
+			}
+
+			for (unsigned j = 0; j < cnt; ++j) {
+				struct radv_amdgpu_cs *cs = radv_amdgpu_cs(cs_array[i + j]);
+				memcpy(ptr, cs->base.buf, 4 * cs->base.cdw);
+				ptr += cs->base.cdw;
+
+			}
+
+			for (unsigned j = 0; j < pad_words; ++j)
+				*ptr++ = pad_word;
+
+			ibs[0].size = size;
+			ibs[0].ib_mc_address = radv_buffer_get_va(bos[0]);
+		}
 
 		r = radv_amdgpu_create_bo_list(cs0->ws, &cs_array[i], cnt,
-		                               (struct radv_amdgpu_winsys_bo*)bo,
-		                               preamble_cs, radv_bo_list, &bo_list);
+			                       (struct radv_amdgpu_winsys_bo **)bos,
+					       number_of_ibs, preamble_cs,
+					       radv_bo_list, &bo_list);
 		if (r) {
 			fprintf(stderr, "amdgpu: buffer list creation failed "
 					"for the sysmem submission (%d)\n", r);
 			return r;
 		}
 
-		ib.size = size;
-		ib.ib_mc_address = radv_buffer_get_va(bo);
+		memset(&request, 0, sizeof(request));
 
 		request.ip_type = cs0->hw_ip;
 		request.ring = queue_idx;
 		request.resources = bo_list;
-		request.number_of_ibs = 1;
-		request.ibs = &ib;
+		request.number_of_ibs = number_of_ibs;
+		request.ibs = ibs;
 		request.fence_info = radv_set_cs_fence(ctx, cs0->hw_ip, queue_idx);
 
 		sem_info->cs_emit_signal = (i == cs_count - cnt) ? emit_signal_sem : false;
@@ -1000,9 +1116,11 @@ static int radv_amdgpu_winsys_cs_submit_sysmem(struct radeon_winsys_ctx *_ctx,
 		if (bo_list)
 			amdgpu_bo_list_destroy(bo_list);
 
-		ws->buffer_destroy(bo);
-		if (r)
-			return r;
+		for (unsigned j = 0; j < number_of_ibs; j++) {
+			ws->buffer_destroy(bos[j]);
+			if (r)
+				return r;
+		}
 
 		i += cnt;
 	}