i965/fs: Use the GRF for UNTYPED_ATOMIC instructions

Signed-off-by: Jason Ekstrand <[email protected]>
Reviewed-by: Matt Turner <[email protected]>

6 files changed, 36 insertions, 25 deletions

diff --git a/src/mesa/drivers/dri/i965/brw_eu.h b/src/mesa/drivers/dri/i965/brw_eu.h
index db40497751a..c14bbd961ab 100644
--- a/src/mesa/drivers/dri/i965/brw_eu.h
+++ b/src/mesa/drivers/dri/i965/brw_eu.h
@@ -390,7 +390,7 @@ void brw_CMP(struct brw_compile *p,
 void
 brw_untyped_atomic(struct brw_compile *p,
                    struct brw_reg dest,
-                   struct brw_reg mrf,
+                   struct brw_reg payload,
                    unsigned atomic_op,
                    unsigned bind_table_index,
                    unsigned msg_length,
diff --git a/src/mesa/drivers/dri/i965/brw_eu_emit.c b/src/mesa/drivers/dri/i965/brw_eu_emit.c
index e4488fc8ead..da3c31623db 100644
--- a/src/mesa/drivers/dri/i965/brw_eu_emit.c
+++ b/src/mesa/drivers/dri/i965/brw_eu_emit.c
@@ -2714,7 +2714,7 @@ brw_set_dp_untyped_atomic_message(struct brw_compile *p,
 void
 brw_untyped_atomic(struct brw_compile *p,
                    struct brw_reg dest,
-                   struct brw_reg mrf,
+                   struct brw_reg payload,
                    unsigned atomic_op,
                    unsigned bind_table_index,
                    unsigned msg_length,
@@ -2723,7 +2723,7 @@ brw_untyped_atomic(struct brw_compile *p,
    brw_inst *insn = brw_next_insn(p, BRW_OPCODE_SEND);
 
    brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UD));
-   brw_set_src0(p, insn, retype(mrf, BRW_REGISTER_TYPE_UD));
+   brw_set_src0(p, insn, retype(payload, BRW_REGISTER_TYPE_UD));
    brw_set_src1(p, insn, brw_imm_d(0));
    brw_set_dp_untyped_atomic_message(
       p, insn, atomic_op, bind_table_index, msg_length, response_length,
diff --git a/src/mesa/drivers/dri/i965/brw_fs.cpp b/src/mesa/drivers/dri/i965/brw_fs.cpp
index 1487e664646..13f673baa34 100644
--- a/src/mesa/drivers/dri/i965/brw_fs.cpp
+++ b/src/mesa/drivers/dri/i965/brw_fs.cpp
@@ -507,6 +507,7 @@ fs_inst::is_send_from_grf() const
    case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
    case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
    case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
+   case SHADER_OPCODE_UNTYPED_ATOMIC:
       return true;
    case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
       return src[1].file == GRF;
@@ -913,6 +914,8 @@ fs_inst::regs_read(fs_visitor *v, int arg) const
 {
    if (is_tex() && arg == 0 && src[0].file == GRF) {
       return mlen;
+   } else if (opcode == SHADER_OPCODE_UNTYPED_ATOMIC && arg == 0) {
+      return mlen;
    }
 
    switch (src[arg].file) {
diff --git a/src/mesa/drivers/dri/i965/brw_fs.h b/src/mesa/drivers/dri/i965/brw_fs.h
index 57c5dc3fe47..7639e2f2b25 100644
--- a/src/mesa/drivers/dri/i965/brw_fs.h
+++ b/src/mesa/drivers/dri/i965/brw_fs.h
@@ -743,6 +743,7 @@ private:
 
    void generate_untyped_atomic(fs_inst *inst,
                                 struct brw_reg dst,
+                                struct brw_reg payload,
                                 struct brw_reg atomic_op,
                                 struct brw_reg surf_index);
 
diff --git a/src/mesa/drivers/dri/i965/brw_fs_generator.cpp b/src/mesa/drivers/dri/i965/brw_fs_generator.cpp
index 7c6f487c67c..9d15c760727 100644
--- a/src/mesa/drivers/dri/i965/brw_fs_generator.cpp
+++ b/src/mesa/drivers/dri/i965/brw_fs_generator.cpp
@@ -1472,6 +1472,7 @@ fs_generator::generate_shader_time_add(fs_inst *inst,
 
 void
 fs_generator::generate_untyped_atomic(fs_inst *inst, struct brw_reg dst,
+                                      struct brw_reg payload,
                                       struct brw_reg atomic_op,
                                       struct brw_reg surf_index)
 {
@@ -1480,9 +1481,8 @@ fs_generator::generate_untyped_atomic(fs_inst *inst, struct brw_reg dst,
           surf_index.file == BRW_IMMEDIATE_VALUE &&
 	  surf_index.type == BRW_REGISTER_TYPE_UD);
 
-   brw_untyped_atomic(p, dst, brw_message_reg(inst->base_mrf),
-                      atomic_op.dw1.ud, surf_index.dw1.ud,
-                      inst->mlen, dispatch_width / 8);
+   brw_untyped_atomic(p, dst, payload, atomic_op.dw1.ud, surf_index.dw1.ud,
+                      inst->mlen, inst->exec_size / 8);
 
    brw_mark_surface_used(prog_data, surf_index.dw1.ud);
 }
@@ -1898,7 +1898,7 @@ fs_generator::generate_code(const cfg_t *cfg)
          break;
 
       case SHADER_OPCODE_UNTYPED_ATOMIC:
-         generate_untyped_atomic(inst, dst, src[0], src[1]);
+         generate_untyped_atomic(inst, dst, src[0], src[1], src[2]);
          break;
 
       case SHADER_OPCODE_UNTYPED_SURFACE_READ:
diff --git a/src/mesa/drivers/dri/i965/brw_fs_visitor.cpp b/src/mesa/drivers/dri/i965/brw_fs_visitor.cpp
index 5822f2293e4..4375ca85b42 100644
--- a/src/mesa/drivers/dri/i965/brw_fs_visitor.cpp
+++ b/src/mesa/drivers/dri/i965/brw_fs_visitor.cpp
@@ -2729,46 +2729,53 @@ fs_visitor::emit_untyped_atomic(unsigned atomic_op, unsigned surf_index,
    bool uses_kill =
       (stage == MESA_SHADER_FRAGMENT) &&
       ((brw_wm_prog_data*) this->prog_data)->uses_kill;
-   const unsigned operand_len = dispatch_width / 8;
-   unsigned mlen = 0;
-   fs_inst *inst;
+   int reg_width = dispatch_width / 8;
+   int length = 0;
 
+   fs_reg *sources = ralloc_array(mem_ctx, fs_reg, 4);
+
+   sources[0] = fs_reg(GRF, virtual_grf_alloc(1), BRW_REGISTER_TYPE_UD);
    /* Initialize the sample mask in the message header. */
-   emit(MOV(brw_uvec_mrf(8, mlen, 0), fs_reg(0u)))
+   emit(MOV(sources[0], fs_reg(0u)))
       ->force_writemask_all = true;
 
    if (uses_kill) {
-      emit(MOV(brw_uvec_mrf(1, mlen, 7), brw_flag_reg(0, 1)))
+      emit(MOV(component(sources[0], 7), brw_flag_reg(0, 1)))
          ->force_writemask_all = true;
    } else {
-      emit(MOV(brw_uvec_mrf(1, mlen, 7),
+      emit(MOV(component(sources[0], 7),
                retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UD)))
          ->force_writemask_all = true;
    }
-
-   mlen++;
+   length++;
 
    /* Set the atomic operation offset. */
-   emit(MOV(brw_uvec_mrf(dispatch_width, mlen, 0), offset));
-   mlen += operand_len;
+   sources[1] = fs_reg(this, glsl_type::uint_type);
+   emit(MOV(sources[1], offset));
+   length++;
 
    /* Set the atomic operation arguments. */
    if (src0.file != BAD_FILE) {
-      emit(MOV(brw_uvec_mrf(dispatch_width, mlen, 0), src0));
-      mlen += operand_len;
+      sources[length] = fs_reg(this, glsl_type::uint_type);
+      emit(MOV(sources[length], src0));
+      length++;
    }
 
    if (src1.file != BAD_FILE) {
-      emit(MOV(brw_uvec_mrf(dispatch_width, mlen, 0), src1));
-      mlen += operand_len;
+      sources[length] = fs_reg(this, glsl_type::uint_type);
+      emit(MOV(sources[length], src1));
+      length++;
    }
 
+   int mlen = 1 + (length - 1) * reg_width;
+   fs_reg src_payload = fs_reg(GRF, virtual_grf_alloc(mlen),
+                               BRW_REGISTER_TYPE_UD);
+   emit(LOAD_PAYLOAD(src_payload, sources, length));
+
    /* Emit the instruction. */
-   inst = emit(SHADER_OPCODE_UNTYPED_ATOMIC, dst,
-               fs_reg(atomic_op), fs_reg(surf_index));
-   inst->base_mrf = 0;
+   fs_inst *inst = emit(SHADER_OPCODE_UNTYPED_ATOMIC, dst, src_payload,
+                        fs_reg(atomic_op), fs_reg(surf_index));
    inst->mlen = mlen;
-   inst->header_present = true;
 }
 
 void