nir/intrinsics: Add more atomic_counter ops

v2: Delete some stray debug code notice by Iago.

v3: Massive rebase on new ir_function_signature::intrinsic_id mechanism.

Signed-off-by: Ian Romanick <[email protected]>
Reviewed-by: Iago Toral Quiroga <[email protected]> [v1]
Acked-by: Ilia Mirkin <[email protected]>

3 files changed, 110 insertions, 5 deletions

diff --git a/src/compiler/glsl/glsl_to_nir.cpp b/src/compiler/glsl/glsl_to_nir.cpp
index eecd9053fe5..18600c484f3 100644
--- a/src/compiler/glsl/glsl_to_nir.cpp
+++ b/src/compiler/glsl/glsl_to_nir.cpp
@@ -620,6 +620,30 @@ nir_visitor::visit(ir_call *ir)
       case ir_intrinsic_atomic_counter_predecrement:
          op = nir_intrinsic_atomic_counter_dec_var;
          break;
+      case ir_intrinsic_atomic_counter_add:
+         op = nir_intrinsic_atomic_counter_add_var;
+         break;
+      case ir_intrinsic_atomic_counter_and:
+         op = nir_intrinsic_atomic_counter_and_var;
+         break;
+      case ir_intrinsic_atomic_counter_or:
+         op = nir_intrinsic_atomic_counter_or_var;
+         break;
+      case ir_intrinsic_atomic_counter_xor:
+         op = nir_intrinsic_atomic_counter_xor_var;
+         break;
+      case ir_intrinsic_atomic_counter_min:
+         op = nir_intrinsic_atomic_counter_min_var;
+         break;
+      case ir_intrinsic_atomic_counter_max:
+         op = nir_intrinsic_atomic_counter_max_var;
+         break;
+      case ir_intrinsic_atomic_counter_exchange:
+         op = nir_intrinsic_atomic_counter_exchange_var;
+         break;
+      case ir_intrinsic_atomic_counter_comp_swap:
+         op = nir_intrinsic_atomic_counter_comp_swap_var;
+         break;
       case ir_intrinsic_image_load:
          op = nir_intrinsic_image_load;
          break;
@@ -771,11 +795,40 @@ nir_visitor::visit(ir_call *ir)
       switch (op) {
       case nir_intrinsic_atomic_counter_read_var:
       case nir_intrinsic_atomic_counter_inc_var:
-      case nir_intrinsic_atomic_counter_dec_var: {
-         ir_dereference *param =
-            (ir_dereference *) ir->actual_parameters.get_head();
-         instr->variables[0] = evaluate_deref(&instr->instr, param);
-         nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
+      case nir_intrinsic_atomic_counter_dec_var:
+      case nir_intrinsic_atomic_counter_add_var:
+      case nir_intrinsic_atomic_counter_min_var:
+      case nir_intrinsic_atomic_counter_max_var:
+      case nir_intrinsic_atomic_counter_and_var:
+      case nir_intrinsic_atomic_counter_or_var:
+      case nir_intrinsic_atomic_counter_xor_var:
+      case nir_intrinsic_atomic_counter_exchange_var:
+      case nir_intrinsic_atomic_counter_comp_swap_var: {
+         /* Set the counter variable dereference. */
+         exec_node *param = ir->actual_parameters.get_head();
+         ir_dereference *counter = (ir_dereference *)param;
+
+         instr->variables[0] = evaluate_deref(&instr->instr, counter);
+         param = param->get_next();
+
+         /* Set the intrinsic destination. */
+         if (ir->return_deref) {
+            nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
+         }
+
+         /* Set the intrinsic parameters. */
+         if (!param->is_tail_sentinel()) {
+            instr->src[0] =
+               nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
+            param = param->get_next();
+         }
+
+         if (!param->is_tail_sentinel()) {
+            instr->src[1] =
+               nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
+            param = param->get_next();
+         }
+
          nir_builder_instr_insert(&b, &instr->instr);
          break;
       }
diff --git a/src/compiler/nir/nir_intrinsics.h b/src/compiler/nir/nir_intrinsics.h
index 1568c2857a4..00bd294b910 100644
--- a/src/compiler/nir/nir_intrinsics.h
+++ b/src/compiler/nir/nir_intrinsics.h
@@ -141,10 +141,24 @@ INTRINSIC(set_vertex_count, 1, ARR(1), false, 0, 0, 0, xx, xx, xx, 0)
 #define ATOMIC(name, flags) \
    INTRINSIC(name##_var, 0, ARR(0), true, 1, 1, 0, xx, xx, xx, flags) \
    INTRINSIC(name, 1, ARR(1), true, 1, 0, 1, BASE, xx, xx, flags)
+#define ATOMIC2(name) \
+   INTRINSIC(name##_var, 1, ARR(1), true, 1, 1, 0, xx, xx, xx, 0) \
+   INTRINSIC(name, 2, ARR(1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
+#define ATOMIC3(name) \
+   INTRINSIC(name##_var, 2, ARR(1, 1), true, 1, 1, 0, xx, xx, xx, 0) \
+   INTRINSIC(name, 3, ARR(1, 1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 
 ATOMIC(atomic_counter_inc, 0)
 ATOMIC(atomic_counter_dec, 0)
 ATOMIC(atomic_counter_read, NIR_INTRINSIC_CAN_ELIMINATE)
+ATOMIC2(atomic_counter_add)
+ATOMIC2(atomic_counter_min)
+ATOMIC2(atomic_counter_max)
+ATOMIC2(atomic_counter_and)
+ATOMIC2(atomic_counter_or)
+ATOMIC2(atomic_counter_xor)
+ATOMIC2(atomic_counter_exchange)
+ATOMIC3(atomic_counter_comp_swap)
 
 /*
  * Image load, store and atomic intrinsics.
diff --git a/src/compiler/nir/nir_lower_atomics.c b/src/compiler/nir/nir_lower_atomics.c
index 04e1febc179..583e2a50c74 100644
--- a/src/compiler/nir/nir_lower_atomics.c
+++ b/src/compiler/nir/nir_lower_atomics.c
@@ -54,6 +54,38 @@ lower_instr(nir_intrinsic_instr *instr,
       op = nir_intrinsic_atomic_counter_dec;
       break;
 
+   case nir_intrinsic_atomic_counter_add_var:
+      op = nir_intrinsic_atomic_counter_add;
+      break;
+
+   case nir_intrinsic_atomic_counter_min_var:
+      op = nir_intrinsic_atomic_counter_min;
+      break;
+
+   case nir_intrinsic_atomic_counter_max_var:
+      op = nir_intrinsic_atomic_counter_max;
+      break;
+
+   case nir_intrinsic_atomic_counter_and_var:
+      op = nir_intrinsic_atomic_counter_and;
+      break;
+
+   case nir_intrinsic_atomic_counter_or_var:
+      op = nir_intrinsic_atomic_counter_or;
+      break;
+
+   case nir_intrinsic_atomic_counter_xor_var:
+      op = nir_intrinsic_atomic_counter_xor;
+      break;
+
+   case nir_intrinsic_atomic_counter_exchange_var:
+      op = nir_intrinsic_atomic_counter_exchange;
+      break;
+
+   case nir_intrinsic_atomic_counter_comp_swap_var:
+      op = nir_intrinsic_atomic_counter_comp_swap;
+      break;
+
    default:
       return;
    }
@@ -120,6 +152,12 @@ lower_instr(nir_intrinsic_instr *instr,
    new_instr->src[0].is_ssa = true;
    new_instr->src[0].ssa = offset_def;
 
+   /* Copy the other sources, if any, from the original instruction to the new
+    * instruction.
+    */
+   for (unsigned i = 0; i < nir_intrinsic_infos[instr->intrinsic].num_srcs; i++)
+      new_instr->src[i + 1] = instr->src[i];
+
    if (instr->dest.is_ssa) {
       nir_ssa_dest_init(&new_instr->instr, &new_instr->dest,
                         instr->dest.ssa.num_components, 32, NULL);