radv: emit the GLC bit for SSBO loads/stores when needed

This fixes some new memory model tests:
dEQP-VK.memory_model.message_passing.core11.u32.coherent.fence_fence.atomicwrite.device.*

Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=108112
Signed-off-by: Samuel Pitoiset <[email protected]>
Reviewed-by: Bas Nieuwenhuizen <[email protected]>

3 files changed, 22 insertions, 8 deletions

diff --git a/src/amd/common/ac_llvm_build.c b/src/amd/common/ac_llvm_build.c
index d11c12fcd13..d0bcfe2dfb3 100644
--- a/src/amd/common/ac_llvm_build.c
+++ b/src/amd/common/ac_llvm_build.c
@@ -1179,7 +1179,8 @@ ac_build_tbuffer_load_short(struct ac_llvm_context *ctx,
 			    LLVMValueRef vindex,
 			    LLVMValueRef voffset,
 				LLVMValueRef soffset,
-				LLVMValueRef immoffset)
+				LLVMValueRef immoffset,
+				LLVMValueRef glc)
 {
 	const char *name = "llvm.amdgcn.tbuffer.load.i32";
 	LLVMTypeRef type = ctx->i32;
@@ -1191,7 +1192,7 @@ ac_build_tbuffer_load_short(struct ac_llvm_context *ctx,
 				immoffset,
 				LLVMConstInt(ctx->i32, V_008F0C_BUF_DATA_FORMAT_16, false),
 				LLVMConstInt(ctx->i32, V_008F0C_BUF_NUM_FORMAT_UINT, false),
-				ctx->i1false,
+				glc,
 				ctx->i1false,
 	};
 	LLVMValueRef res = ac_build_intrinsic(ctx, name, type, params, 9, 0);
diff --git a/src/amd/common/ac_llvm_build.h b/src/amd/common/ac_llvm_build.h
index 65235fb7352..5c3e221b2cc 100644
--- a/src/amd/common/ac_llvm_build.h
+++ b/src/amd/common/ac_llvm_build.h
@@ -274,7 +274,8 @@ ac_build_tbuffer_load_short(struct ac_llvm_context *ctx,
 			    LLVMValueRef vindex,
 			    LLVMValueRef voffset,
 				LLVMValueRef soffset,
-				LLVMValueRef immoffset);
+				LLVMValueRef immoffset,
+				LLVMValueRef glc);
 
 LLVMValueRef
 ac_get_thread_id(struct ac_llvm_context *ctx);
diff --git a/src/amd/common/ac_nir_to_llvm.c b/src/amd/common/ac_nir_to_llvm.c
index ffc64a79d95..e0a8e04cf39 100644
--- a/src/amd/common/ac_nir_to_llvm.c
+++ b/src/amd/common/ac_nir_to_llvm.c
@@ -1456,6 +1456,11 @@ static void visit_store_ssbo(struct ac_nir_context *ctx,
 	LLVMValueRef src_data = get_src(ctx, instr->src[0]);
 	int elem_size_bytes = ac_get_elem_bits(&ctx->ac, LLVMTypeOf(src_data)) / 8;
 	unsigned writemask = nir_intrinsic_write_mask(instr);
+	enum gl_access_qualifier access = nir_intrinsic_access(instr);
+	LLVMValueRef glc = ctx->ac.i1false;
+
+	if (access & (ACCESS_VOLATILE | ACCESS_COHERENT))
+		glc = ctx->ac.i1true;
 
 	LLVMValueRef rsrc = ctx->abi->load_ssbo(ctx->abi,
 				        get_src(ctx, instr->src[1]), true);
@@ -1512,7 +1517,7 @@ static void visit_store_ssbo(struct ac_nir_context *ctx,
 				ctx->ac.i32_0,
 				LLVMConstInt(ctx->ac.i32, 2, false), // dfmt (= 16bit)
 				LLVMConstInt(ctx->ac.i32, 4, false), // nfmt (= uint)
-				ctx->ac.i1false,
+				glc,
 				ctx->ac.i1false,
 			};
 			ac_build_intrinsic(&ctx->ac, store_name,
@@ -1540,7 +1545,7 @@ static void visit_store_ssbo(struct ac_nir_context *ctx,
 				rsrc,
 				ctx->ac.i32_0, /* vindex */
 				offset,
-				ctx->ac.i1false,  /* glc */
+				glc,
 				ctx->ac.i1false,  /* slc */
 			};
 			ac_build_intrinsic(&ctx->ac, store_name,
@@ -1613,6 +1618,11 @@ static LLVMValueRef visit_load_buffer(struct ac_nir_context *ctx,
 	int elem_size_bytes = instr->dest.ssa.bit_size / 8;
 	int num_components = instr->num_components;
 	int num_bytes = num_components * elem_size_bytes;
+	enum gl_access_qualifier access = nir_intrinsic_access(instr);
+	LLVMValueRef glc = ctx->ac.i1false;
+
+	if (access & (ACCESS_VOLATILE | ACCESS_COHERENT))
+		glc = ctx->ac.i1true;
 
 	for (int i = 0; i < num_bytes; i += load_bytes) {
 		load_bytes = MIN2(num_bytes - i, 16);
@@ -1631,7 +1641,8 @@ static LLVMValueRef visit_load_buffer(struct ac_nir_context *ctx,
 								   vindex,
 								   offset,
 								   ctx->ac.i32_0,
-								   immoffset);
+								   immoffset,
+								   glc);
 		} else {
 			switch (load_bytes) {
 			case 16:
@@ -1655,7 +1666,7 @@ static LLVMValueRef visit_load_buffer(struct ac_nir_context *ctx,
 				rsrc,
 				vindex,
 				LLVMBuildAdd(ctx->ac.builder, offset, immoffset, ""),
-				ctx->ac.i1false,
+				glc,
 				ctx->ac.i1false,
 			};
 			results[idx] = ac_build_intrinsic(&ctx->ac, load_name, data_type, params, 5, 0);
@@ -1709,7 +1720,8 @@ static LLVMValueRef visit_load_ubo_buffer(struct ac_nir_context *ctx,
 								 ctx->ac.i32_0,
 								 offset,
 								 ctx->ac.i32_0,
-								 LLVMConstInt(ctx->ac.i32, 2 * i, 0));
+								 LLVMConstInt(ctx->ac.i32, 2 * i, 0),
+								 ctx->ac.i1false);
 		}
 		ret = ac_build_gather_values(&ctx->ac, results, num_components);
 	} else {