3 files changed, 72 insertions, 9 deletions

diff --git a/src/amd/common/ac_llvm_build.h b/src/amd/common/ac_llvm_build.h
index f4cee667153..98f856106d6 100644
--- a/src/amd/common/ac_llvm_build.h
+++ b/src/amd/common/ac_llvm_build.h
@@ -26,7 +26,7 @@
 #define AC_LLVM_BUILD_H
 
 #include <stdbool.h>
-#include <llvm-c/TargetMachine.h>
+#include <llvm-c/Core.h>
 #include "compiler/nir/nir.h"
 #include "amd_family.h"
 
@@ -694,6 +694,14 @@ ac_build_ddxy_interp(struct ac_llvm_context *ctx, LLVMValueRef interp_ij);
 LLVMValueRef
 ac_build_load_helper_invocation(struct ac_llvm_context *ctx);
 
+LLVMValueRef ac_build_atomic_rmw(struct ac_llvm_context *ctx, LLVMAtomicRMWBinOp op,
+				 LLVMValueRef ptr, LLVMValueRef val,
+				 const char *sync_scope);
+
+LLVMValueRef ac_build_atomic_cmp_xchg(struct ac_llvm_context *ctx, LLVMValueRef ptr,
+				      LLVMValueRef cmp, LLVMValueRef val,
+				      const char *sync_scope);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/amd/common/ac_llvm_helper.cpp b/src/amd/common/ac_llvm_helper.cpp
index dcfb8008546..e5030c6f472 100644
--- a/src/amd/common/ac_llvm_helper.cpp
+++ b/src/amd/common/ac_llvm_helper.cpp
@@ -31,6 +31,7 @@
 
 #include "ac_binary.h"
 #include "ac_llvm_util.h"
+#include "ac_llvm_build.h"
 
 #include <llvm-c/Core.h>
 #include <llvm/Target/TargetMachine.h>
@@ -167,3 +168,61 @@ void ac_enable_global_isel(LLVMTargetMachineRef tm)
 {
   reinterpret_cast<llvm::TargetMachine*>(tm)->setGlobalISel(true);
 }
+
+LLVMValueRef ac_build_atomic_rmw(struct ac_llvm_context *ctx, LLVMAtomicRMWBinOp op,
+				 LLVMValueRef ptr, LLVMValueRef val,
+				 const char *sync_scope) {
+	llvm::AtomicRMWInst::BinOp binop;
+	switch (op) {
+	case LLVMAtomicRMWBinOpXchg:
+		binop = llvm::AtomicRMWInst::Xchg;
+		break;
+	case LLVMAtomicRMWBinOpAdd:
+		binop = llvm::AtomicRMWInst::Add;
+		break;
+	case LLVMAtomicRMWBinOpSub:
+		binop = llvm::AtomicRMWInst::Sub;
+		break;
+	case LLVMAtomicRMWBinOpAnd:
+		binop = llvm::AtomicRMWInst::And;
+		break;
+	case LLVMAtomicRMWBinOpNand:
+		binop = llvm::AtomicRMWInst::Nand;
+		break;
+	case LLVMAtomicRMWBinOpOr:
+		binop = llvm::AtomicRMWInst::Or;
+		break;
+	case LLVMAtomicRMWBinOpXor:
+		binop = llvm::AtomicRMWInst::Xor;
+		break;
+	case LLVMAtomicRMWBinOpMax:
+		binop = llvm::AtomicRMWInst::Max;
+		break;
+	case LLVMAtomicRMWBinOpMin:
+		binop = llvm::AtomicRMWInst::Min;
+		break;
+	case LLVMAtomicRMWBinOpUMax:
+		binop = llvm::AtomicRMWInst::UMax;
+		break;
+	case LLVMAtomicRMWBinOpUMin:
+		binop = llvm::AtomicRMWInst::UMin;
+		break;
+	default:
+		unreachable(!"invalid LLVMAtomicRMWBinOp");
+	   break;
+	}
+	unsigned SSID = llvm::unwrap(ctx->context)->getOrInsertSyncScopeID(sync_scope);
+	return llvm::wrap(llvm::unwrap(ctx->builder)->CreateAtomicRMW(
+		binop, llvm::unwrap(ptr), llvm::unwrap(val),
+		llvm::AtomicOrdering::SequentiallyConsistent, SSID));
+}
+
+LLVMValueRef ac_build_atomic_cmp_xchg(struct ac_llvm_context *ctx, LLVMValueRef ptr,
+				      LLVMValueRef cmp, LLVMValueRef val,
+				      const char *sync_scope) {
+	unsigned SSID = llvm::unwrap(ctx->context)->getOrInsertSyncScopeID(sync_scope);
+	return llvm::wrap(llvm::unwrap(ctx->builder)->CreateAtomicCmpXchg(
+			  llvm::unwrap(ptr), llvm::unwrap(cmp), llvm::unwrap(val),
+			  llvm::AtomicOrdering::SequentiallyConsistent,
+			  llvm::AtomicOrdering::SequentiallyConsistent, SSID));
+}
diff --git a/src/amd/common/ac_nir_to_llvm.c b/src/amd/common/ac_nir_to_llvm.c
index 92aea7df9bf..c92eaaca31d 100644
--- a/src/amd/common/ac_nir_to_llvm.c
+++ b/src/amd/common/ac_nir_to_llvm.c
@@ -2897,14 +2897,12 @@ static LLVMValueRef visit_var_atomic(struct ac_nir_context *ctx,
 	LLVMValueRef result;
 	LLVMValueRef src = get_src(ctx, instr->src[src_idx]);
 
+	const char *sync_scope = HAVE_LLVM >= 0x0900 ? "workgroup-one-as" : "workgroup";
+
 	if (instr->intrinsic == nir_intrinsic_shared_atomic_comp_swap ||
 	    instr->intrinsic == nir_intrinsic_deref_atomic_comp_swap) {
 		LLVMValueRef src1 = get_src(ctx, instr->src[src_idx + 1]);
-		result = LLVMBuildAtomicCmpXchg(ctx->ac.builder,
-						ptr, src, src1,
-						LLVMAtomicOrderingSequentiallyConsistent,
-						LLVMAtomicOrderingSequentiallyConsistent,
-						false);
+		result = ac_build_atomic_cmp_xchg(&ctx->ac, ptr, src, src1, sync_scope);
 		result = LLVMBuildExtractValue(ctx->ac.builder, result, 0, "");
 	} else {
 		LLVMAtomicRMWBinOp op;
@@ -2949,9 +2947,7 @@ static LLVMValueRef visit_var_atomic(struct ac_nir_context *ctx,
 			return NULL;
 		}
 
-		result = LLVMBuildAtomicRMW(ctx->ac.builder, op, ptr, ac_to_integer(&ctx->ac, src),
-					    LLVMAtomicOrderingSequentiallyConsistent,
-					    false);
+		result = ac_build_atomic_rmw(&ctx->ac, op, ptr, ac_to_integer(&ctx->ac, src), sync_scope);
 	}
 	return result;
 }