nir: Add floating point atomic min, max, and compare-swap instrinsics

Signed-off-by: Ian Romanick <[email protected]>
Reviewed-by: Caio Marcelo de Oliveira Filho <[email protected]>

4 files changed, 50 insertions, 8 deletions

diff --git a/src/compiler/glsl/glsl_to_nir.cpp b/src/compiler/glsl/glsl_to_nir.cpp
index aba8d70a8a0..a53000f47e9 100644
--- a/src/compiler/glsl/glsl_to_nir.cpp
+++ b/src/compiler/glsl/glsl_to_nir.cpp
@@ -709,6 +709,8 @@ nir_visitor::visit(ir_call *ir)
             op = nir_intrinsic_ssbo_atomic_imin;
          else if (ir->return_deref->type == glsl_type::uint_type)
             op = nir_intrinsic_ssbo_atomic_umin;
+         else if (ir->return_deref->type == glsl_type::float_type)
+            op = nir_intrinsic_ssbo_atomic_fmin;
          else
             unreachable("Invalid type");
          break;
@@ -718,6 +720,8 @@ nir_visitor::visit(ir_call *ir)
             op = nir_intrinsic_ssbo_atomic_imax;
          else if (ir->return_deref->type == glsl_type::uint_type)
             op = nir_intrinsic_ssbo_atomic_umax;
+         else if (ir->return_deref->type == glsl_type::float_type)
+            op = nir_intrinsic_ssbo_atomic_fmax;
          else
             unreachable("Invalid type");
          break;
@@ -725,7 +729,9 @@ nir_visitor::visit(ir_call *ir)
          op = nir_intrinsic_ssbo_atomic_exchange;
          break;
       case ir_intrinsic_ssbo_atomic_comp_swap:
-         op = nir_intrinsic_ssbo_atomic_comp_swap;
+         op = ir->return_deref->type->is_integer_32_64()
+            ? nir_intrinsic_ssbo_atomic_comp_swap
+            : nir_intrinsic_ssbo_atomic_fcomp_swap;
          break;
       case ir_intrinsic_shader_clock:
          op = nir_intrinsic_shader_clock;
@@ -777,6 +783,8 @@ nir_visitor::visit(ir_call *ir)
             op = nir_intrinsic_shared_atomic_imin;
          else if (ir->return_deref->type == glsl_type::uint_type)
             op = nir_intrinsic_shared_atomic_umin;
+         else if (ir->return_deref->type == glsl_type::float_type)
+            op = nir_intrinsic_shared_atomic_fmin;
          else
             unreachable("Invalid type");
          break;
@@ -786,6 +794,8 @@ nir_visitor::visit(ir_call *ir)
             op = nir_intrinsic_shared_atomic_imax;
          else if (ir->return_deref->type == glsl_type::uint_type)
             op = nir_intrinsic_shared_atomic_umax;
+         else if (ir->return_deref->type == glsl_type::float_type)
+            op = nir_intrinsic_shared_atomic_fmax;
          else
             unreachable("Invalid type");
          break;
@@ -793,7 +803,9 @@ nir_visitor::visit(ir_call *ir)
          op = nir_intrinsic_shared_atomic_exchange;
          break;
       case ir_intrinsic_shared_atomic_comp_swap:
-         op = nir_intrinsic_shared_atomic_comp_swap;
+         op = ir->return_deref->type->is_integer_32_64()
+            ? nir_intrinsic_shared_atomic_comp_swap
+            : nir_intrinsic_shared_atomic_fcomp_swap;
          break;
       case ir_intrinsic_vote_any:
          op = nir_intrinsic_vote_any;
@@ -1042,7 +1054,10 @@ nir_visitor::visit(ir_call *ir)
       case nir_intrinsic_ssbo_atomic_xor:
       case nir_intrinsic_ssbo_atomic_exchange:
       case nir_intrinsic_ssbo_atomic_comp_swap:
-      case nir_intrinsic_ssbo_atomic_fadd: {
+      case nir_intrinsic_ssbo_atomic_fadd:
+      case nir_intrinsic_ssbo_atomic_fmin:
+      case nir_intrinsic_ssbo_atomic_fmax:
+      case nir_intrinsic_ssbo_atomic_fcomp_swap: {
          int param_count = ir->actual_parameters.length();
          assert(param_count == 3 || param_count == 4);
 
@@ -1063,7 +1078,8 @@ nir_visitor::visit(ir_call *ir)
 
          /* data2 parameter (only with atomic_comp_swap) */
          if (param_count == 4) {
-            assert(op == nir_intrinsic_ssbo_atomic_comp_swap);
+            assert(op == nir_intrinsic_ssbo_atomic_comp_swap ||
+                   op == nir_intrinsic_ssbo_atomic_fcomp_swap);
             param = param->get_next();
             inst = (ir_instruction *) param;
             instr->src[3] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
@@ -1126,7 +1142,10 @@ nir_visitor::visit(ir_call *ir)
       case nir_intrinsic_shared_atomic_xor:
       case nir_intrinsic_shared_atomic_exchange:
       case nir_intrinsic_shared_atomic_comp_swap:
-      case nir_intrinsic_shared_atomic_fadd: {
+      case nir_intrinsic_shared_atomic_fadd:
+      case nir_intrinsic_shared_atomic_fmin:
+      case nir_intrinsic_shared_atomic_fmax:
+      case nir_intrinsic_shared_atomic_fcomp_swap:  {
          int param_count = ir->actual_parameters.length();
          assert(param_count == 2 || param_count == 3);
 
@@ -1142,7 +1161,8 @@ nir_visitor::visit(ir_call *ir)
 
          /* data2 parameter (only with atomic_comp_swap) */
          if (param_count == 3) {
-            assert(op == nir_intrinsic_shared_atomic_comp_swap);
+            assert(op == nir_intrinsic_shared_atomic_comp_swap ||
+                   op == nir_intrinsic_shared_atomic_fcomp_swap);
             param = param->get_next();
             inst = (ir_instruction *) param;
             instr->src[2] =
diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py
index 836cb34d74a..8c9bc3bbc9a 100644
--- a/src/compiler/nir/nir_intrinsics.py
+++ b/src/compiler/nir/nir_intrinsics.py
@@ -359,6 +359,9 @@ intrinsic("deref_atomic_xor",  src_comp=[1, 1], dest_comp=1)
 intrinsic("deref_atomic_exchange", src_comp=[1, 1], dest_comp=1)
 intrinsic("deref_atomic_comp_swap", src_comp=[1, 1, 1], dest_comp=1)
 intrinsic("deref_atomic_fadd",  src_comp=[1, 1], dest_comp=1)
+intrinsic("deref_atomic_fmin",  src_comp=[1, 1], dest_comp=1)
+intrinsic("deref_atomic_fmax",  src_comp=[1, 1], dest_comp=1)
+intrinsic("deref_atomic_fcomp_swap", src_comp=[1, 1, 1], dest_comp=1)
 
 # SSBO atomic intrinsics
 #
@@ -385,7 +388,10 @@ intrinsic("ssbo_atomic_or",   src_comp=[1, 1, 1], dest_comp=1)
 intrinsic("ssbo_atomic_xor",  src_comp=[1, 1, 1], dest_comp=1)
 intrinsic("ssbo_atomic_exchange", src_comp=[1, 1, 1], dest_comp=1)
 intrinsic("ssbo_atomic_comp_swap", src_comp=[1, 1, 1, 1], dest_comp=1)
-intrinsic("ssbo_atomic_fadd",  src_comp=[1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_fadd", src_comp=[1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_fmin", src_comp=[1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_fmax", src_comp=[1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_fcomp_swap", src_comp=[1, 1, 1, 1], dest_comp=1)
 
 # CS shared variable atomic intrinsics
 #
@@ -412,6 +418,9 @@ intrinsic("shared_atomic_xor",  src_comp=[1, 1], dest_comp=1, indices=[BASE])
 intrinsic("shared_atomic_exchange", src_comp=[1, 1], dest_comp=1, indices=[BASE])
 intrinsic("shared_atomic_comp_swap", src_comp=[1, 1, 1], dest_comp=1, indices=[BASE])
 intrinsic("shared_atomic_fadd",  src_comp=[1, 1], dest_comp=1, indices=[BASE])
+intrinsic("shared_atomic_fmin",  src_comp=[1, 1], dest_comp=1, indices=[BASE])
+intrinsic("shared_atomic_fmax",  src_comp=[1, 1], dest_comp=1, indices=[BASE])
+intrinsic("shared_atomic_fcomp_swap", src_comp=[1, 1, 1], dest_comp=1, indices=[BASE])
 
 def system_value(name, dest_comp, indices=[]):
     intrinsic("load_" + name, [], dest_comp, indices,
diff --git a/src/compiler/nir/nir_lower_atomics_to_ssbo.c b/src/compiler/nir/nir_lower_atomics_to_ssbo.c
index 72686624329..c165a03b141 100644
--- a/src/compiler/nir/nir_lower_atomics_to_ssbo.c
+++ b/src/compiler/nir/nir_lower_atomics_to_ssbo.c
@@ -58,6 +58,9 @@ lower_instr(nir_intrinsic_instr *instr, unsigned ssbo_offset, nir_builder *b)
    case nir_intrinsic_ssbo_atomic_exchange:
    case nir_intrinsic_ssbo_atomic_comp_swap:
    case nir_intrinsic_ssbo_atomic_fadd:
+   case nir_intrinsic_ssbo_atomic_fmin:
+   case nir_intrinsic_ssbo_atomic_fmax:
+   case nir_intrinsic_ssbo_atomic_fcomp_swap:
    case nir_intrinsic_store_ssbo:
    case nir_intrinsic_load_ssbo:
    case nir_intrinsic_get_buffer_size:
@@ -140,7 +143,8 @@ lower_instr(nir_intrinsic_instr *instr, unsigned ssbo_offset, nir_builder *b)
       new_instr->src[0] = nir_src_for_ssa(buffer);
       nir_src_copy(&new_instr->src[1], &instr->src[0], new_instr);
       nir_src_copy(&new_instr->src[2], &instr->src[1], new_instr);
-      if (op == nir_intrinsic_ssbo_atomic_comp_swap)
+      if (op == nir_intrinsic_ssbo_atomic_comp_swap ||
+          op == nir_intrinsic_ssbo_atomic_fcomp_swap)
          nir_src_copy(&new_instr->src[3], &instr->src[2], new_instr);
       break;
    }
diff --git a/src/compiler/nir/nir_lower_io.c b/src/compiler/nir/nir_lower_io.c
index 3d4451e3602..c418433aa52 100644
--- a/src/compiler/nir/nir_lower_io.c
+++ b/src/compiler/nir/nir_lower_io.c
@@ -281,6 +281,9 @@ lower_atomic(nir_intrinsic_instr *intrin, struct lower_io_state *state,
    OP(atomic_or)
    OP(atomic_xor)
    OP(atomic_fadd)
+   OP(atomic_fmin)
+   OP(atomic_fmax)
+   OP(atomic_fcomp_swap)
 #undef OP
    default:
       unreachable("Invalid atomic");
@@ -382,6 +385,9 @@ nir_lower_io_block(nir_block *block,
       case nir_intrinsic_deref_atomic_exchange:
       case nir_intrinsic_deref_atomic_comp_swap:
       case nir_intrinsic_deref_atomic_fadd:
+      case nir_intrinsic_deref_atomic_fmin:
+      case nir_intrinsic_deref_atomic_fmax:
+      case nir_intrinsic_deref_atomic_fcomp_swap:
          /* We can lower the io for this nir instrinsic */
          break;
       case nir_intrinsic_interp_deref_at_centroid:
@@ -444,6 +450,9 @@ nir_lower_io_block(nir_block *block,
       case nir_intrinsic_deref_atomic_exchange:
       case nir_intrinsic_deref_atomic_comp_swap:
       case nir_intrinsic_deref_atomic_fadd:
+      case nir_intrinsic_deref_atomic_fmin:
+      case nir_intrinsic_deref_atomic_fmax:
+      case nir_intrinsic_deref_atomic_fcomp_swap:
          assert(vertex_index == NULL);
          replacement = lower_atomic(intrin, state, var, offset);
          break;