r600/sfn: Add support for atomic instructions

v2: fix compilation with gcc-6

Signed-off-by: Gert Wollny <[email protected]>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/merge_requests/3225>

6 files changed, 228 insertions, 0 deletions

diff --git a/src/gallium/drivers/r600/Makefile.sources b/src/gallium/drivers/r600/Makefile.sources
index f5cf78110a4..a17997e6eae 100644
--- a/src/gallium/drivers/r600/Makefile.sources
+++ b/src/gallium/drivers/r600/Makefile.sources
@@ -101,6 +101,8 @@ CXX_SOURCES = \
 	sfn/sfn_emitaluinstruction.h \
 	sfn/sfn_emitinstruction.cpp \
 	sfn/sfn_emitinstruction.h \
+	sfn/sfn_emitssboinstruction.cpp \
+	sfn/sfn_emitssboinstruction.h \
 	sfn/sfn_emittexinstruction.cpp \
 	sfn/sfn_emittexinstruction.h \
 	sfn/sfn_emitinstruction.h \
diff --git a/src/gallium/drivers/r600/meson.build b/src/gallium/drivers/r600/meson.build
index 5080741c308..ecdf8504032 100644
--- a/src/gallium/drivers/r600/meson.build
+++ b/src/gallium/drivers/r600/meson.build
@@ -118,6 +118,8 @@ files_r600 = files(
   'sfn/sfn_emitaluinstruction.h',
   'sfn/sfn_emitinstruction.cpp',
   'sfn/sfn_emitinstruction.h',
+  'sfn/sfn_emitssboinstruction.cpp',
+  'sfn/sfn_emitssboinstruction.h',
   'sfn/sfn_emittexinstruction.cpp',
   'sfn/sfn_emittexinstruction.h',
   'sfn/sfn_emitinstruction.h',
diff --git a/src/gallium/drivers/r600/sfn/sfn_emitssboinstruction.cpp b/src/gallium/drivers/r600/sfn/sfn_emitssboinstruction.cpp
new file mode 100644
index 00000000000..0b5bc5fa15d
--- /dev/null
+++ b/src/gallium/drivers/r600/sfn/sfn_emitssboinstruction.cpp
@@ -0,0 +1,177 @@
+#include "sfn_emitssboinstruction.h"
+
+#include "sfn_instruction_fetch.h"
+#include "sfn_instruction_gds.h"
+#include "sfn_instruction_misc.h"
+#include "../r600_pipe.h"
+
+namespace r600 {
+
+bool EmitSSBOInstruction::do_emit(nir_instr* instr)
+{
+   const nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+   switch (intr->intrinsic) {
+   case nir_intrinsic_atomic_counter_add:
+   case nir_intrinsic_atomic_counter_and:
+   case nir_intrinsic_atomic_counter_exchange:
+   case nir_intrinsic_atomic_counter_max:
+   case nir_intrinsic_atomic_counter_min:
+   case nir_intrinsic_atomic_counter_or:
+   case nir_intrinsic_atomic_counter_xor:
+   case nir_intrinsic_atomic_counter_comp_swap:
+      return emit_atomic(intr);
+   case nir_intrinsic_atomic_counter_read:
+   case nir_intrinsic_atomic_counter_post_dec:
+      return emit_unary_atomic(intr);
+   case nir_intrinsic_atomic_counter_inc:
+      return emit_atomic_inc(intr);
+   case nir_intrinsic_atomic_counter_pre_dec:
+      return emit_atomic_pre_dec(intr);
+   default:
+      return false;
+   }
+}
+
+bool EmitSSBOInstruction::emit_atomic(const nir_intrinsic_instr* instr)
+{
+   ESDOp op = get_opcode(instr->intrinsic);
+
+   if (DS_OP_INVALID == op)
+      return false;
+
+   GPRVector dest = make_dest(instr);
+
+   int base = nir_intrinsic_base(instr);
+
+   PValue uav_id = from_nir(instr->src[0], 0);
+
+   PValue value = from_nir_with_fetch_constant(instr->src[1], 0);
+
+   GDSInstr *ir = nullptr;
+   if (instr->intrinsic == nir_intrinsic_atomic_counter_comp_swap)  {
+      PValue value2 = from_nir_with_fetch_constant(instr->src[1], 1);
+      ir = new GDSInstr(op, dest, value, value2, uav_id, base);
+   } else {
+      ir = new GDSInstr(op, dest, value, uav_id, base);
+   }
+
+   emit_instruction(ir);
+   return true;
+}
+
+bool EmitSSBOInstruction::emit_unary_atomic(const nir_intrinsic_instr* instr)
+{
+   ESDOp op = get_opcode(instr->intrinsic);
+
+   if (DS_OP_INVALID == op)
+      return false;
+
+   GPRVector dest = make_dest(instr);
+
+   PValue uav_id = from_nir(instr->src[0], 0);
+
+   auto ir = new GDSInstr(op, dest, uav_id, nir_intrinsic_base(instr));
+
+   emit_instruction(ir);
+   return true;
+}
+
+ESDOp EmitSSBOInstruction::get_opcode(const nir_intrinsic_op opcode)
+{
+   switch (opcode) {
+   case nir_intrinsic_atomic_counter_add:
+      return DS_OP_ADD_RET;
+   case nir_intrinsic_atomic_counter_and:
+      return DS_OP_AND_RET;
+   case nir_intrinsic_atomic_counter_exchange:
+      return DS_OP_XCHG_RET;
+   case nir_intrinsic_atomic_counter_inc:
+      return DS_OP_INC_RET;
+   case nir_intrinsic_atomic_counter_max:
+      return DS_OP_MAX_UINT_RET;
+   case nir_intrinsic_atomic_counter_min:
+      return DS_OP_MIN_UINT_RET;
+   case nir_intrinsic_atomic_counter_or:
+      return DS_OP_OR_RET;
+   case nir_intrinsic_atomic_counter_read:
+      return DS_OP_READ_RET;
+   case nir_intrinsic_atomic_counter_xor:
+      return DS_OP_XOR_RET;
+   case nir_intrinsic_atomic_counter_post_dec:
+      return DS_OP_DEC_RET;
+   case nir_intrinsic_atomic_counter_comp_swap:
+      return DS_OP_CMP_XCHG_RET;
+   case nir_intrinsic_atomic_counter_pre_dec:
+   default:
+      return DS_OP_INVALID;
+   }
+}
+
+
+bool EmitSSBOInstruction::emit_atomic_add(const nir_intrinsic_instr* instr)
+{
+   GPRVector dest = make_dest(instr);
+
+   PValue value = from_nir_with_fetch_constant(instr->src[1], 0);
+
+   PValue uav_id = from_nir(instr->src[0], 0);
+
+   auto ir = new GDSInstr(DS_OP_ADD_RET, dest, value, uav_id,
+                          nir_intrinsic_base(instr));
+
+   emit_instruction(ir);
+   return true;
+}
+
+bool EmitSSBOInstruction::emit_atomic_inc(const nir_intrinsic_instr* instr)
+{
+   GPRVector dest = make_dest(instr);
+
+   PValue uav_id = from_nir(instr->src[0], 0);
+
+
+   if (!m_atomic_limit) {
+      int one_tmp = allocate_temp_register();
+      m_atomic_limit = PValue(new GPRValue(one_tmp, 0));
+      emit_instruction(new AluInstruction(op1_mov, m_atomic_limit,
+                       PValue(new LiteralValue(0xffffffff)),
+                       {alu_write, alu_last_instr}));
+   }
+
+   auto ir = new GDSInstr(DS_OP_INC_RET, dest, m_atomic_limit, uav_id,
+                          nir_intrinsic_base(instr));
+   emit_instruction(ir);
+   return true;
+}
+
+bool EmitSSBOInstruction::emit_atomic_pre_dec(const nir_intrinsic_instr *instr)
+{
+   GPRVector dest = make_dest(instr);
+
+   PValue uav_id = from_nir(instr->src[0], 0);
+
+   int one_tmp = allocate_temp_register();
+   PValue value(new GPRValue(one_tmp, 0));
+   emit_instruction(new AluInstruction(op1_mov, value,  Value::one_i,
+                    {alu_write, alu_last_instr}));
+
+   auto ir = new GDSInstr(DS_OP_SUB_RET, dest, value, uav_id,
+                          nir_intrinsic_base(instr));
+   emit_instruction(ir);
+
+   ir = new GDSInstr(DS_OP_READ_RET, dest, uav_id, nir_intrinsic_base(instr));
+   emit_instruction(ir);
+
+   return true;
+}
+
+GPRVector EmitSSBOInstruction::make_dest(const nir_intrinsic_instr* ir)
+{
+   GPRVector::Values v;
+   int i;
+   for (i = 0; i < 4; ++i)
+      v[i] = from_nir(ir->dest, i);
+   return GPRVector(v);
+}
+
+}
diff --git a/src/gallium/drivers/r600/sfn/sfn_emitssboinstruction.h b/src/gallium/drivers/r600/sfn/sfn_emitssboinstruction.h
new file mode 100644
index 00000000000..300f2ec2e92
--- /dev/null
+++ b/src/gallium/drivers/r600/sfn/sfn_emitssboinstruction.h
@@ -0,0 +1,29 @@
+#ifndef SFN_EMITSSBOINSTRUCTION_H
+#define SFN_EMITSSBOINSTRUCTION_H
+
+#include "sfn_emitinstruction.h"
+
+namespace r600 {
+
+class EmitSSBOInstruction: public EmitInstruction {
+public:
+   using EmitInstruction::EmitInstruction;
+private:
+   bool do_emit(nir_instr *instr);
+
+   bool emit_atomic(const nir_intrinsic_instr* instr);
+   bool emit_unary_atomic(const nir_intrinsic_instr* instr);
+   bool emit_atomic_add(const nir_intrinsic_instr* instr);
+   bool emit_atomic_inc(const nir_intrinsic_instr* instr);
+   bool emit_atomic_pre_dec(const nir_intrinsic_instr* instr);
+
+   ESDOp get_opcode(nir_intrinsic_op opcode);
+
+   GPRVector make_dest(const nir_intrinsic_instr* instr);
+
+   PValue m_atomic_limit;
+};
+
+}
+
+#endif // SFN_EMITSSBOINSTRUCTION_H
diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp b/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp
index 0ee6bbd6313..7ea7bc69893 100644
--- a/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp
+++ b/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp
@@ -62,6 +62,7 @@ ShaderFromNirProcessor::ShaderFromNirProcessor(pipe_shader_type ptype,
    m_sh_info(sh_info),
    m_tex_instr(*this),
    m_alu_instr(*this),
+   m_ssbo_instr(*this),
    m_pending_else(nullptr),
    m_scratch_size(scratch_size),
    m_next_hwatomic_loc(0),
@@ -449,6 +450,21 @@ bool ShaderFromNirProcessor::emit_intrinsic_instruction(nir_intrinsic_instr* ins
       return emit_discard_if(instr);
    case nir_intrinsic_load_ubo_r600:
       return emit_load_ubo(instr);
+   case nir_intrinsic_atomic_counter_add:
+   case nir_intrinsic_atomic_counter_and:
+   case nir_intrinsic_atomic_counter_exchange:
+   case nir_intrinsic_atomic_counter_max:
+   case nir_intrinsic_atomic_counter_min:
+   case nir_intrinsic_atomic_counter_or:
+   case nir_intrinsic_atomic_counter_xor:
+   case nir_intrinsic_atomic_counter_comp_swap:
+   case nir_intrinsic_atomic_counter_read:
+   case nir_intrinsic_atomic_counter_post_dec:
+   case nir_intrinsic_atomic_counter_inc:
+   case nir_intrinsic_atomic_counter_pre_dec:
+      m_sel.info.writes_memory = true;
+      return m_ssbo_instr.emit(&instr->instr);
+      break;
    case nir_intrinsic_copy_deref:
    case nir_intrinsic_load_constant:
    case nir_intrinsic_load_input:
diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_base.h b/src/gallium/drivers/r600/sfn/sfn_shader_base.h
index 3cd9b971e4a..5b027329093 100644
--- a/src/gallium/drivers/r600/sfn/sfn_shader_base.h
+++ b/src/gallium/drivers/r600/sfn/sfn_shader_base.h
@@ -40,6 +40,7 @@
 #include "sfn_instruction_cf.h"
 #include "sfn_emittexinstruction.h"
 #include "sfn_emitaluinstruction.h"
+#include "sfn_emitssboinstruction.h"
 
 #include <vector>
 #include <set>
@@ -174,6 +175,7 @@ private:
 
    EmitTexInstruction m_tex_instr;
    EmitAluInstruction m_alu_instr;
+   EmitSSBOInstruction m_ssbo_instr;
    OutputRegisterMap m_output_register_map;
 
    IfElseInstruction *m_pending_else;