i965: Fix the untyped surface opcodes to deal with indirect surface access.

Change brw_untyped_atomic() and brw_untyped_surface_read() to take the
surface index as a register instead of a constant and to use
brw_send_indirect_message() to emit the indirect variant of send with
a dynamically calculated message descriptor.  This will be required to
support variable indexing of image arrays for
ARB_shader_image_load_store.

Acked-by: Kenneth Graunke <[email protected]>
Reviewed-by: Topi Pohjolainen <[email protected]>

4 files changed, 99 insertions, 82 deletions

diff --git a/src/mesa/drivers/dri/i965/brw_eu.h b/src/mesa/drivers/dri/i965/brw_eu.h
index b8c63a372d5..1d0e5634027 100644
--- a/src/mesa/drivers/dri/i965/brw_eu.h
+++ b/src/mesa/drivers/dri/i965/brw_eu.h
@@ -401,18 +401,18 @@ void brw_CMP(struct brw_codegen *p,
 
 void
 brw_untyped_atomic(struct brw_codegen *p,
-                   struct brw_reg dest,
+                   struct brw_reg dst,
                    struct brw_reg payload,
+                   struct brw_reg surface,
                    unsigned atomic_op,
-                   unsigned bind_table_index,
                    unsigned msg_length,
                    bool response_expected);
 
 void
 brw_untyped_surface_read(struct brw_codegen *p,
-                         struct brw_reg dest,
-                         struct brw_reg mrf,
-                         unsigned bind_table_index,
+                         struct brw_reg dst,
+                         struct brw_reg payload,
+                         struct brw_reg surface,
                          unsigned msg_length,
                          unsigned num_channels);
 
diff --git a/src/mesa/drivers/dri/i965/brw_eu_emit.c b/src/mesa/drivers/dri/i965/brw_eu_emit.c
index 1ffd332f57e..3ae1a1a56ac 100644
--- a/src/mesa/drivers/dri/i965/brw_eu_emit.c
+++ b/src/mesa/drivers/dri/i965/brw_eu_emit.c
@@ -2524,6 +2524,48 @@ brw_send_indirect_message(struct brw_codegen *p,
    return setup;
 }
 
+static struct brw_inst *
+brw_send_indirect_surface_message(struct brw_codegen *p,
+                                  unsigned sfid,
+                                  struct brw_reg dst,
+                                  struct brw_reg payload,
+                                  struct brw_reg surface,
+                                  unsigned message_len,
+                                  unsigned response_len,
+                                  bool header_present)
+{
+   const struct brw_device_info *devinfo = p->devinfo;
+   struct brw_inst *insn;
+
+   if (surface.file != BRW_IMMEDIATE_VALUE) {
+      struct brw_reg addr = retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD);
+
+      brw_push_insn_state(p);
+      brw_set_default_access_mode(p, BRW_ALIGN_1);
+      brw_set_default_mask_control(p, BRW_MASK_DISABLE);
+      brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
+
+      /* Mask out invalid bits from the surface index to avoid hangs e.g. when
+       * some surface array is accessed out of bounds.
+       */
+      insn = brw_AND(p, addr,
+                     suboffset(vec1(retype(surface, BRW_REGISTER_TYPE_UD)),
+                               BRW_GET_SWZ(surface.dw1.bits.swizzle, 0)),
+                     brw_imm_ud(0xff));
+
+      brw_pop_insn_state(p);
+
+      surface = addr;
+   }
+
+   insn = brw_send_indirect_message(p, sfid, dst, payload, surface);
+   brw_inst_set_mlen(devinfo, insn, message_len);
+   brw_inst_set_rlen(devinfo, insn, response_len);
+   brw_inst_set_header_present(devinfo, insn, header_present);
+
+   return insn;
+}
+
 static int
 brw_find_next_block_end(struct brw_codegen *p, int start_offset)
 {
@@ -2747,25 +2789,16 @@ static void
 brw_set_dp_untyped_atomic_message(struct brw_codegen *p,
                                   brw_inst *insn,
                                   unsigned atomic_op,
-                                  unsigned bind_table_index,
-                                  unsigned msg_length,
-                                  unsigned response_length,
-                                  bool header_present)
+                                  bool response_expected)
 {
    const struct brw_device_info *devinfo = p->devinfo;
-
    unsigned msg_control =
       atomic_op | /* Atomic Operation Type: BRW_AOP_* */
-      (response_length ? 1 << 5 : 0); /* Return data expected */
+      (response_expected ? 1 << 5 : 0); /* Return data expected */
 
    if (devinfo->gen >= 8 || devinfo->is_haswell) {
-      brw_set_message_descriptor(p, insn, HSW_SFID_DATAPORT_DATA_CACHE_1,
-                                 msg_length, response_length,
-                                 header_present, false);
-
-
-      if (brw_inst_access_mode(devinfo, insn) == BRW_ALIGN_1) {
-         if (brw_inst_exec_size(devinfo, insn) != BRW_EXECUTE_16)
+      if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
+         if (!p->compressed)
             msg_control |= 1 << 4; /* SIMD8 mode */
 
          brw_inst_set_dp_msg_type(devinfo, insn,
@@ -2775,30 +2808,29 @@ brw_set_dp_untyped_atomic_message(struct brw_codegen *p,
             HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2);
       }
    } else {
-      brw_set_message_descriptor(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE,
-                                 msg_length, response_length,
-                                 header_present, false);
-
-      brw_inst_set_dp_msg_type(devinfo, insn, GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP);
+      brw_inst_set_dp_msg_type(devinfo, insn,
+                               GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP);
 
-      if (brw_inst_exec_size(devinfo, insn) != BRW_EXECUTE_16)
+      if (!p->compressed)
          msg_control |= 1 << 4; /* SIMD8 mode */
    }
 
-   brw_inst_set_binding_table_index(devinfo, insn, bind_table_index);
    brw_inst_set_dp_msg_control(devinfo, insn, msg_control);
 }
 
 void
 brw_untyped_atomic(struct brw_codegen *p,
-                   struct brw_reg dest,
+                   struct brw_reg dst,
                    struct brw_reg payload,
+                   struct brw_reg surface,
                    unsigned atomic_op,
-                   unsigned bind_table_index,
                    unsigned msg_length,
                    bool response_expected)
 {
    const struct brw_device_info *devinfo = p->devinfo;
+   const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
+                          HSW_SFID_DATAPORT_DATA_CACHE_1 :
+                          GEN7_SFID_DATAPORT_DATA_CACHE);
    const bool align1 = brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1;
    /* Mask out unused components -- This is especially important in Align16
     * mode on generations that don't have native support for SIMD4x2 atomics,
@@ -2807,79 +2839,59 @@ brw_untyped_atomic(struct brw_codegen *p,
     * uninitialized Y, Z and W coordinates of the payload.
     */
    const unsigned mask = align1 ? WRITEMASK_XYZW : WRITEMASK_X;
-   brw_inst *insn = brw_next_insn(p, BRW_OPCODE_SEND);
-
-   brw_set_dest(p, insn, retype(brw_writemask(dest, mask),
-                                BRW_REGISTER_TYPE_UD));
-   brw_set_src0(p, insn, retype(payload, BRW_REGISTER_TYPE_UD));
-   brw_set_src1(p, insn, brw_imm_d(0));
-   brw_set_dp_untyped_atomic_message(
-      p, insn, atomic_op, bind_table_index, msg_length,
+   struct brw_inst *insn = brw_send_indirect_surface_message(
+      p, sfid, brw_writemask(dst, mask), payload, surface, msg_length,
       brw_surface_payload_size(p, response_expected,
                                devinfo->gen >= 8 || devinfo->is_haswell, true),
       align1);
+
+   brw_set_dp_untyped_atomic_message(
+      p, insn, atomic_op, response_expected);
 }
 
 static void
 brw_set_dp_untyped_surface_read_message(struct brw_codegen *p,
-                                        brw_inst *insn,
-                                        unsigned bind_table_index,
-                                        unsigned msg_length,
-                                        unsigned response_length,
-                                        unsigned num_channels,
-                                        bool header_present)
+                                        struct brw_inst *insn,
+                                        unsigned num_channels)
 {
    const struct brw_device_info *devinfo = p->devinfo;
-   const unsigned dispatch_width =
-      (brw_inst_exec_size(devinfo, insn) == BRW_EXECUTE_16 ? 16 : 8);
-
-   if (devinfo->gen >= 8 || devinfo->is_haswell) {
-      brw_set_message_descriptor(p, insn, HSW_SFID_DATAPORT_DATA_CACHE_1,
-                                 msg_length, response_length,
-                                 header_present, false);
-
-      brw_inst_set_dp_msg_type(devinfo, insn,
-                               HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ);
-   } else {
-      brw_set_message_descriptor(p, insn, GEN7_SFID_DATAPORT_DATA_CACHE,
-                                 msg_length, response_length,
-                                 header_present, false);
-
-      brw_inst_set_dp_msg_type(devinfo, insn,
-                               GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ);
-   }
-
    /* Set mask of 32-bit channels to drop. */
-   unsigned msg_control = (0xf & (0xf << num_channels));
+   unsigned msg_control = 0xf & (0xf << num_channels);
 
-   if (brw_inst_access_mode(devinfo, insn) == BRW_ALIGN_1) {
-      if (dispatch_width == 16)
+   if (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1) {
+      if (p->compressed)
          msg_control |= 1 << 4; /* SIMD16 mode */
       else
          msg_control |= 2 << 4; /* SIMD8 mode */
    }
 
-   brw_inst_set_binding_table_index(devinfo, insn, bind_table_index);
+   brw_inst_set_dp_msg_type(devinfo, insn,
+                            (devinfo->gen >= 8 || devinfo->is_haswell ?
+                             HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ :
+                             GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ));
    brw_inst_set_dp_msg_control(devinfo, insn, msg_control);
 }
 
 void
 brw_untyped_surface_read(struct brw_codegen *p,
-                         struct brw_reg dest,
-                         struct brw_reg mrf,
-                         unsigned bind_table_index,
+                         struct brw_reg dst,
+                         struct brw_reg payload,
+                         struct brw_reg surface,
                          unsigned msg_length,
                          unsigned num_channels)
 {
    const struct brw_device_info *devinfo = p->devinfo;
-   brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
+   const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
+                          HSW_SFID_DATAPORT_DATA_CACHE_1 :
+                          GEN7_SFID_DATAPORT_DATA_CACHE);
+   const bool align1 = (brw_inst_access_mode(devinfo, p->current) == BRW_ALIGN_1);
+   struct brw_inst *insn = brw_send_indirect_surface_message(
+      p, sfid, dst, payload, surface, msg_length,
+      brw_surface_payload_size(p, num_channels, true, true),
+      align1);
 
-   brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UD));
-   brw_set_src0(p, insn, retype(mrf, BRW_REGISTER_TYPE_UD));
    brw_set_dp_untyped_surface_read_message(
-      p, insn, bind_table_index, msg_length,
-      brw_surface_payload_size(p, num_channels, true, true),
-      num_channels, brw_inst_access_mode(devinfo, insn) == BRW_ALIGN_1);
+      p, insn, num_channels);
 }
 
 void
@@ -2930,13 +2942,16 @@ void brw_shader_time_add(struct brw_codegen *p,
                          struct brw_reg payload,
                          uint32_t surf_index)
 {
+   const unsigned sfid = (p->devinfo->gen >= 8 || p->devinfo->is_haswell ?
+                          HSW_SFID_DATAPORT_DATA_CACHE_1 :
+                          GEN7_SFID_DATAPORT_DATA_CACHE);
    assert(p->devinfo->gen >= 7);
 
    brw_push_insn_state(p);
    brw_set_default_access_mode(p, BRW_ALIGN_1);
    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
+   brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
    brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
-   brw_pop_insn_state(p);
 
    /* We use brw_vec1_reg and unmasked because we want to increment the given
     * offset only once.
@@ -2945,8 +2960,10 @@ void brw_shader_time_add(struct brw_codegen *p,
                                       BRW_ARF_NULL, 0));
    brw_set_src0(p, send, brw_vec1_reg(payload.file,
                                       payload.nr, 0));
-   brw_set_dp_untyped_atomic_message(p, send, BRW_AOP_ADD, surf_index,
-                                     2 /* message length */,
-                                     0 /* response length */,
-                                     false /* header present */);
+   brw_set_src1(p, send, brw_imm_ud(0));
+   brw_set_message_descriptor(p, send, sfid, 2, 0, false, false);
+   brw_inst_set_binding_table_index(p->devinfo, send, surf_index);
+   brw_set_dp_untyped_atomic_message(p, send, BRW_AOP_ADD, false);
+
+   brw_pop_insn_state(p);
 }
diff --git a/src/mesa/drivers/dri/i965/brw_fs_generator.cpp b/src/mesa/drivers/dri/i965/brw_fs_generator.cpp
index 9ddba3a26dc..e28f4430be8 100644
--- a/src/mesa/drivers/dri/i965/brw_fs_generator.cpp
+++ b/src/mesa/drivers/dri/i965/brw_fs_generator.cpp
@@ -1540,7 +1540,7 @@ fs_generator::generate_untyped_atomic(fs_inst *inst, struct brw_reg dst,
 	  surf_index.type == BRW_REGISTER_TYPE_UD);
 
    brw_untyped_atomic(p, dst, payload,
-                      atomic_op.dw1.ud, surf_index.dw1.ud,
+                      surf_index, atomic_op.dw1.ud,
                       inst->mlen, true);
 
    brw_mark_surface_used(prog_data, surf_index.dw1.ud);
@@ -1554,7 +1554,7 @@ fs_generator::generate_untyped_surface_read(fs_inst *inst, struct brw_reg dst,
    assert(surf_index.file == BRW_IMMEDIATE_VALUE &&
 	  surf_index.type == BRW_REGISTER_TYPE_UD);
 
-   brw_untyped_surface_read(p, dst, payload, surf_index.dw1.ud, inst->mlen, 1);
+   brw_untyped_surface_read(p, dst, payload, surf_index, inst->mlen, 1);
 
    brw_mark_surface_used(prog_data, surf_index.dw1.ud);
 }
diff --git a/src/mesa/drivers/dri/i965/brw_vec4_generator.cpp b/src/mesa/drivers/dri/i965/brw_vec4_generator.cpp
index 07b008f0c8f..b9ab0123b48 100644
--- a/src/mesa/drivers/dri/i965/brw_vec4_generator.cpp
+++ b/src/mesa/drivers/dri/i965/brw_vec4_generator.cpp
@@ -1122,7 +1122,7 @@ vec4_generator::generate_untyped_atomic(vec4_instruction *inst,
 	  surf_index.type == BRW_REGISTER_TYPE_UD);
 
    brw_untyped_atomic(p, dst, brw_message_reg(inst->base_mrf),
-                      atomic_op.dw1.ud, surf_index.dw1.ud,
+                      surf_index, atomic_op.dw1.ud,
                       inst->mlen, true);
 
    brw_mark_surface_used(&prog_data->base, surf_index.dw1.ud);
@@ -1137,7 +1137,7 @@ vec4_generator::generate_untyped_surface_read(vec4_instruction *inst,
 	  surf_index.type == BRW_REGISTER_TYPE_UD);
 
    brw_untyped_surface_read(p, dst, brw_message_reg(inst->base_mrf),
-                            surf_index.dw1.ud, inst->mlen, 1);
+                            surf_index, inst->mlen, 1);
 
    brw_mark_surface_used(&prog_data->base, surf_index.dw1.ud);
 }