nir: Add floating point atomic add instrinsics

Signed-off-by: Ian Romanick <[email protected]>
Reviewed-by: Caio Marcelo de Oliveira Filho <[email protected]>

5 files changed, 22 insertions, 5 deletions

diff --git a/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c b/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c
index aae64c9f3f9..9ff5708f503 100644
--- a/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c
+++ b/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c
@@ -240,6 +240,7 @@ lower_intrinsic(nir_intrinsic_instr *instr,
        instr->intrinsic == nir_intrinsic_image_deref_atomic_xor ||
        instr->intrinsic == nir_intrinsic_image_deref_atomic_exchange ||
        instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap ||
+       instr->intrinsic == nir_intrinsic_image_deref_atomic_fadd ||
        instr->intrinsic == nir_intrinsic_image_deref_size) {
 
       b->cursor = nir_before_instr(&instr->instr);
diff --git a/src/compiler/glsl/glsl_to_nir.cpp b/src/compiler/glsl/glsl_to_nir.cpp
index b1d1da89111..aba8d70a8a0 100644
--- a/src/compiler/glsl/glsl_to_nir.cpp
+++ b/src/compiler/glsl/glsl_to_nir.cpp
@@ -650,7 +650,9 @@ nir_visitor::visit(ir_call *ir)
          op = nir_intrinsic_image_deref_store;
          break;
       case ir_intrinsic_image_atomic_add:
-         op = nir_intrinsic_image_deref_atomic_add;
+         op = ir->return_deref->type->is_integer_32_64()
+            ? nir_intrinsic_image_deref_atomic_add
+            : nir_intrinsic_image_deref_atomic_fadd;
          break;
       case ir_intrinsic_image_atomic_min:
          op = nir_intrinsic_image_deref_atomic_min;
@@ -689,7 +691,8 @@ nir_visitor::visit(ir_call *ir)
          op = nir_intrinsic_load_ssbo;
          break;
       case ir_intrinsic_ssbo_atomic_add:
-         op = nir_intrinsic_ssbo_atomic_add;
+         op = ir->return_deref->type->is_integer_32_64()
+            ? nir_intrinsic_ssbo_atomic_add : nir_intrinsic_ssbo_atomic_fadd;
          break;
       case ir_intrinsic_ssbo_atomic_and:
          op = nir_intrinsic_ssbo_atomic_and;
@@ -755,7 +758,9 @@ nir_visitor::visit(ir_call *ir)
          op = nir_intrinsic_store_shared;
          break;
       case ir_intrinsic_shared_atomic_add:
-         op = nir_intrinsic_shared_atomic_add;
+         op = ir->return_deref->type->is_integer_32_64()
+            ? nir_intrinsic_shared_atomic_add
+            : nir_intrinsic_shared_atomic_fadd;
          break;
       case ir_intrinsic_shared_atomic_and:
          op = nir_intrinsic_shared_atomic_and;
@@ -865,6 +870,7 @@ nir_visitor::visit(ir_call *ir)
       case nir_intrinsic_image_deref_atomic_xor:
       case nir_intrinsic_image_deref_atomic_exchange:
       case nir_intrinsic_image_deref_atomic_comp_swap:
+      case nir_intrinsic_image_deref_atomic_fadd:
       case nir_intrinsic_image_deref_samples:
       case nir_intrinsic_image_deref_size: {
          nir_ssa_undef_instr *instr_undef =
@@ -1035,7 +1041,8 @@ nir_visitor::visit(ir_call *ir)
       case nir_intrinsic_ssbo_atomic_or:
       case nir_intrinsic_ssbo_atomic_xor:
       case nir_intrinsic_ssbo_atomic_exchange:
-      case nir_intrinsic_ssbo_atomic_comp_swap: {
+      case nir_intrinsic_ssbo_atomic_comp_swap:
+      case nir_intrinsic_ssbo_atomic_fadd: {
          int param_count = ir->actual_parameters.length();
          assert(param_count == 3 || param_count == 4);
 
@@ -1118,7 +1125,8 @@ nir_visitor::visit(ir_call *ir)
       case nir_intrinsic_shared_atomic_or:
       case nir_intrinsic_shared_atomic_xor:
       case nir_intrinsic_shared_atomic_exchange:
-      case nir_intrinsic_shared_atomic_comp_swap: {
+      case nir_intrinsic_shared_atomic_comp_swap:
+      case nir_intrinsic_shared_atomic_fadd: {
          int param_count = ir->actual_parameters.length();
          assert(param_count == 2 || param_count == 3);
 
diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py
index d688a57181b..836cb34d74a 100644
--- a/src/compiler/nir/nir_intrinsics.py
+++ b/src/compiler/nir/nir_intrinsics.py
@@ -307,6 +307,7 @@ intrinsic("image_deref_atomic_or",   src_comp=[1, 4, 1, 1], dest_comp=1)
 intrinsic("image_deref_atomic_xor",  src_comp=[1, 4, 1, 1], dest_comp=1)
 intrinsic("image_deref_atomic_exchange",  src_comp=[1, 4, 1, 1], dest_comp=1)
 intrinsic("image_deref_atomic_comp_swap", src_comp=[1, 4, 1, 1, 1], dest_comp=1)
+intrinsic("image_deref_atomic_fadd",  src_comp=[1, 4, 1, 1], dest_comp=1)
 intrinsic("image_deref_size",    src_comp=[1], dest_comp=0, flags=[CAN_ELIMINATE, CAN_REORDER])
 intrinsic("image_deref_samples", src_comp=[1], dest_comp=1, flags=[CAN_ELIMINATE, CAN_REORDER])
 
@@ -357,6 +358,7 @@ intrinsic("deref_atomic_or",   src_comp=[1, 1], dest_comp=1)
 intrinsic("deref_atomic_xor",  src_comp=[1, 1], dest_comp=1)
 intrinsic("deref_atomic_exchange", src_comp=[1, 1], dest_comp=1)
 intrinsic("deref_atomic_comp_swap", src_comp=[1, 1, 1], dest_comp=1)
+intrinsic("deref_atomic_fadd",  src_comp=[1, 1], dest_comp=1)
 
 # SSBO atomic intrinsics
 #
@@ -383,6 +385,7 @@ intrinsic("ssbo_atomic_or",   src_comp=[1, 1, 1], dest_comp=1)
 intrinsic("ssbo_atomic_xor",  src_comp=[1, 1, 1], dest_comp=1)
 intrinsic("ssbo_atomic_exchange", src_comp=[1, 1, 1], dest_comp=1)
 intrinsic("ssbo_atomic_comp_swap", src_comp=[1, 1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_fadd",  src_comp=[1, 1, 1], dest_comp=1)
 
 # CS shared variable atomic intrinsics
 #
@@ -408,6 +411,7 @@ intrinsic("shared_atomic_or",   src_comp=[1, 1], dest_comp=1, indices=[BASE])
 intrinsic("shared_atomic_xor",  src_comp=[1, 1], dest_comp=1, indices=[BASE])
 intrinsic("shared_atomic_exchange", src_comp=[1, 1], dest_comp=1, indices=[BASE])
 intrinsic("shared_atomic_comp_swap", src_comp=[1, 1, 1], dest_comp=1, indices=[BASE])
+intrinsic("shared_atomic_fadd",  src_comp=[1, 1], dest_comp=1, indices=[BASE])
 
 def system_value(name, dest_comp, indices=[]):
     intrinsic("load_" + name, [], dest_comp, indices,
diff --git a/src/compiler/nir/nir_lower_atomics_to_ssbo.c b/src/compiler/nir/nir_lower_atomics_to_ssbo.c
index 6ebd3632288..72686624329 100644
--- a/src/compiler/nir/nir_lower_atomics_to_ssbo.c
+++ b/src/compiler/nir/nir_lower_atomics_to_ssbo.c
@@ -57,6 +57,7 @@ lower_instr(nir_intrinsic_instr *instr, unsigned ssbo_offset, nir_builder *b)
    case nir_intrinsic_ssbo_atomic_xor:
    case nir_intrinsic_ssbo_atomic_exchange:
    case nir_intrinsic_ssbo_atomic_comp_swap:
+   case nir_intrinsic_ssbo_atomic_fadd:
    case nir_intrinsic_store_ssbo:
    case nir_intrinsic_load_ssbo:
    case nir_intrinsic_get_buffer_size:
diff --git a/src/compiler/nir/nir_lower_io.c b/src/compiler/nir/nir_lower_io.c
index 9500a3091c4..3d4451e3602 100644
--- a/src/compiler/nir/nir_lower_io.c
+++ b/src/compiler/nir/nir_lower_io.c
@@ -280,6 +280,7 @@ lower_atomic(nir_intrinsic_instr *intrin, struct lower_io_state *state,
    OP(atomic_and)
    OP(atomic_or)
    OP(atomic_xor)
+   OP(atomic_fadd)
 #undef OP
    default:
       unreachable("Invalid atomic");
@@ -380,6 +381,7 @@ nir_lower_io_block(nir_block *block,
       case nir_intrinsic_deref_atomic_xor:
       case nir_intrinsic_deref_atomic_exchange:
       case nir_intrinsic_deref_atomic_comp_swap:
+      case nir_intrinsic_deref_atomic_fadd:
          /* We can lower the io for this nir instrinsic */
          break;
       case nir_intrinsic_interp_deref_at_centroid:
@@ -441,6 +443,7 @@ nir_lower_io_block(nir_block *block,
       case nir_intrinsic_deref_atomic_xor:
       case nir_intrinsic_deref_atomic_exchange:
       case nir_intrinsic_deref_atomic_comp_swap:
+      case nir_intrinsic_deref_atomic_fadd:
          assert(vertex_index == NULL);
          replacement = lower_atomic(intrin, state, var, offset);
          break;