i965: Add a new representation for Broadwell shader instructions.

Broadwell significantly changes the EU instruction encoding.  Many of
the fields got moved to different bit positions; some even got split
in two.

With so many changes, it was infeasible to continue using struct
brw_instruction.  We needed a new representation.

This new approach is a bit different: rather than a struct, I created a
class that has four DWords, and helper functions that read/write various
bits.  This has several advantages:

1. We can create several different names for the same bits.  For
   example, conditional modifiers, SFID for SEND instructions, and the
   MATH instruction's function opcode are all stored in bits 27:24.

   In each situation, we can use the appropriate setter function:
   set_sfid(), set_math_function(), or set_cond_modifier().  This
   is much easier to follow.

2. Since the fields are expressed using the original 128-bit numbers,
   the code to create the getter/setter functions follows the table in
   the documentation very closely.

To aid in debugging, I've enabled -fkeep-inline-functions when building
gen8_instruction.c.  Otherwise, these functions cannot be called by
gdb, making it insanely difficult to print out anything.

Kenneth Graunke wrote most of this code.  Damien Lespiau ported it to
C99.  Xiang Haihao added media fields.  Zhao Yakui added indirect
addressing support.  Eric Anholt added an assertion to make sure that
values fit in the alloted number of bits.

v2: Update for brw_reg_type_to_hw_type(), which necessitates passing
    brw_context pointers around everywhere.

Signed-off-by: Kenneth Graunke <[email protected]>
Signed-off-by: Damien Lespiau <[email protected]>
Signed-off-by: Xiang, Haihao <[email protected]>
Signed-off-by: Zhao Yakui <[email protected]>
Reviewed-by: Eric Anholt <[email protected]>
Acked-by: Matt Turner <[email protected]>

4 files changed, 835 insertions, 0 deletions

diff --git a/src/mesa/drivers/dri/i965/Makefile.am b/src/mesa/drivers/dri/i965/Makefile.am
index a2c3b32da88..24e226f7529 100644
--- a/src/mesa/drivers/dri/i965/Makefile.am
+++ b/src/mesa/drivers/dri/i965/Makefile.am
@@ -38,6 +38,8 @@ AM_CFLAGS = \
 
 AM_CXXFLAGS = $(AM_CFLAGS)
 
+gen8_instruction_CFLAGS = $(AM_CFLAGS) -fkeep-inline-functions
+
 noinst_LTLIBRARIES = libi965_dri.la
 libi965_dri_la_SOURCES = $(i965_FILES)
 libi965_dri_la_LIBADD = $(INTEL_LIBS)
diff --git a/src/mesa/drivers/dri/i965/Makefile.sources b/src/mesa/drivers/dri/i965/Makefile.sources
index 43f152e4d45..be25981480c 100644
--- a/src/mesa/drivers/dri/i965/Makefile.sources
+++ b/src/mesa/drivers/dri/i965/Makefile.sources
@@ -139,4 +139,5 @@ i965_FILES = \
 	gen7_vs_state.c \
 	gen7_wm_state.c \
 	gen7_wm_surface_state.c \
+	gen8_instruction.c \
         $()
diff --git a/src/mesa/drivers/dri/i965/gen8_instruction.c b/src/mesa/drivers/dri/i965/gen8_instruction.c
new file mode 100644
index 00000000000..476b3a91130
--- /dev/null
+++ b/src/mesa/drivers/dri/i965/gen8_instruction.c
@@ -0,0 +1,433 @@
+/*
+ * Copyright © 2012 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+/**
+ * @file gen8_instruction.c
+ *
+ * A representation of a Gen8+ EU instruction, with helper methods to get
+ * and set various fields.  This is the actual hardware format.
+ */
+
+#include "brw_defines.h"
+#include "gen8_instruction.h"
+
+static void
+gen8_convert_mrf_to_grf(struct brw_reg *reg)
+{
+   /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
+    * "The send with EOT should use register space R112-R127 for <src>. This is
+    *  to enable loading of a new thread into the same slot while the message
+    *  with EOT for current thread is pending dispatch."
+    *
+    * Since we're pretending to have 16 MRFs anyway, we may as well use the
+    * registers required for messages with EOT.
+    */
+   if (reg->file == BRW_MESSAGE_REGISTER_FILE) {
+      reg->file = BRW_GENERAL_REGISTER_FILE;
+      reg->nr += GEN7_MRF_HACK_START;
+   }
+}
+
+void
+gen8_set_dst(const struct brw_context *brw,
+             struct gen8_instruction *inst,
+             struct brw_reg reg)
+{
+   gen8_convert_mrf_to_grf(&reg);
+
+   if (reg.file == BRW_GENERAL_REGISTER_FILE)
+      assert(reg.nr < BRW_MAX_GRF);
+
+   gen8_set_dst_reg_file(inst, reg.file);
+   gen8_set_dst_reg_type(inst, brw_reg_type_to_hw_type(brw, reg.type, reg.file));
+   gen8_set_dst_address_mode(inst, reg.address_mode);
+
+   if (reg.address_mode == BRW_ADDRESS_DIRECT) {
+      gen8_set_dst_da_reg_nr(inst, reg.nr);
+
+      if (gen8_access_mode(inst) == BRW_ALIGN_1) {
+         /* Set Dst.SubRegNum[4:0] */
+         gen8_set_dst_da1_subreg_nr(inst, reg.subnr);
+
+         /* Set Dst.HorzStride */
+         if (reg.hstride == BRW_HORIZONTAL_STRIDE_0)
+            reg.hstride = BRW_HORIZONTAL_STRIDE_1;
+         gen8_set_dst_da1_hstride(inst, reg.hstride);
+      } else {
+         /* Align16 SubRegNum only has a single bit (bit 4; bits 3:0 MBZ). */
+         assert(reg.subnr == 0 || reg.subnr == 16);
+         gen8_set_dst_da16_subreg_nr(inst, reg.subnr >> 4);
+         gen8_set_da16_writemask(inst, reg.dw1.bits.writemask);
+      }
+   } else {
+      /* Indirect addressing */
+      assert(gen8_access_mode(inst) == BRW_ALIGN_1);
+
+      /* Set Dst.HorzStride */
+      if (reg.hstride == BRW_HORIZONTAL_STRIDE_0)
+         reg.hstride = BRW_HORIZONTAL_STRIDE_1;
+      gen8_set_dst_da1_hstride(inst, reg.hstride);
+      gen8_set_dst_ia1_subreg_nr(inst, reg.subnr);
+      gen8_set_dst_ia1_addr_imm(inst, reg.dw1.bits.indirect_offset);
+   }
+
+   /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
+    * or 16 (SIMD16), as that's normally correct.  However, when dealing with
+    * small registers, we automatically reduce it to match the register size.
+    */
+   if (reg.width < BRW_EXECUTE_8)
+      gen8_set_exec_size(inst, reg.width);
+}
+
+static void
+gen8_validate_reg(struct gen8_instruction *inst, struct brw_reg reg)
+{
+   int hstride_for_reg[] = {0, 1, 2, 4};
+   int vstride_for_reg[] = {0, 1, 2, 4, 8, 16, 32, 64, 128, 256};
+   int width_for_reg[] = {1, 2, 4, 8, 16};
+   int execsize_for_reg[] = {1, 2, 4, 8, 16};
+   int width, hstride, vstride, execsize;
+
+   if (reg.file == BRW_IMMEDIATE_VALUE) {
+      /* TODO: check immediate vectors */
+      return;
+   }
+
+   if (reg.file == BRW_ARCHITECTURE_REGISTER_FILE)
+      return;
+
+   assert(reg.hstride >= 0 && reg.hstride < ARRAY_SIZE(hstride_for_reg));
+   hstride = hstride_for_reg[reg.hstride];
+
+   if (reg.vstride == 0xf) {
+      vstride = -1;
+   } else {
+      assert(reg.vstride >= 0 && reg.vstride < ARRAY_SIZE(vstride_for_reg));
+      vstride = vstride_for_reg[reg.vstride];
+   }
+
+   assert(reg.width >= 0 && reg.width < ARRAY_SIZE(width_for_reg));
+   width = width_for_reg[reg.width];
+
+   assert(gen8_exec_size(inst) >= 0 &&
+          gen8_exec_size(inst) < ARRAY_SIZE(execsize_for_reg));
+   execsize = execsize_for_reg[gen8_exec_size(inst)];
+
+   /* Restrictions from 3.3.10: Register Region Restrictions. */
+   /* 3. */
+   assert(execsize >= width);
+
+   /* 4. */
+   if (execsize == width && hstride != 0) {
+      assert(vstride == -1 || vstride == width * hstride);
+   }
+
+   /* 5. */
+   if (execsize == width && hstride == 0) {
+      /* no restriction on vstride. */
+   }
+
+   /* 6. */
+   if (width == 1) {
+      assert(hstride == 0);
+   }
+
+   /* 7. */
+   if (execsize == 1 && width == 1) {
+      assert(hstride == 0);
+      assert(vstride == 0);
+   }
+
+   /* 8. */
+   if (vstride == 0 && hstride == 0) {
+      assert(width == 1);
+   }
+
+   /* 10. Check destination issues. */
+}
+
+void
+gen8_set_src0(const struct brw_context *brw,
+              struct gen8_instruction *inst,
+              struct brw_reg reg)
+{
+   gen8_convert_mrf_to_grf(&reg);
+
+   if (reg.file == BRW_GENERAL_REGISTER_FILE)
+      assert(reg.nr < BRW_MAX_GRF);
+
+   gen8_validate_reg(inst, reg);
+
+   gen8_set_src0_reg_file(inst, reg.file);
+   gen8_set_src0_reg_type(inst,
+                          brw_reg_type_to_hw_type(brw, reg.type, reg.file));
+   gen8_set_src0_abs(inst, reg.abs);
+   gen8_set_src0_negate(inst, reg.negate);
+
+   if (reg.file == BRW_IMMEDIATE_VALUE) {
+      inst->data[3] = reg.dw1.ud;
+
+      /* Required to set some fields in src1 as well: */
+      gen8_set_src1_reg_file(inst, BRW_ARCHITECTURE_REGISTER_FILE);
+      gen8_set_src1_reg_type(inst,
+                             brw_reg_type_to_hw_type(brw, reg.type, reg.file));
+      return;
+   }
+
+   gen8_set_src0_address_mode(inst, reg.address_mode);
+
+   if (reg.address_mode == BRW_ADDRESS_DIRECT) {
+      gen8_set_src0_da_reg_nr(inst, reg.nr);
+
+      if (gen8_access_mode(inst) == BRW_ALIGN_1) {
+         /* Set Src0.SubRegNum[4:0] */
+         gen8_set_src0_da1_subreg_nr(inst, reg.subnr);
+
+         if (reg.width == BRW_WIDTH_1 && gen8_exec_size(inst) == BRW_EXECUTE_1) {
+            gen8_set_src0_da1_hstride(inst, BRW_HORIZONTAL_STRIDE_0);
+            gen8_set_src0_vert_stride(inst, BRW_VERTICAL_STRIDE_0);
+         } else {
+            gen8_set_src0_da1_hstride(inst, reg.hstride);
+            gen8_set_src0_vert_stride(inst, reg.vstride);
+         }
+         gen8_set_src0_da1_width(inst, reg.width);
+
+      } else {
+         /* Align16 SubRegNum only has a single bit (bit 4; bits 3:0 MBZ). */
+         assert(reg.subnr == 0 || reg.subnr == 16);
+         gen8_set_src0_da16_subreg_nr(inst, reg.subnr >> 4);
+
+         gen8_set_src0_da16_swiz_x(inst,
+                                   BRW_GET_SWZ(reg.dw1.bits.swizzle,
+                                               BRW_CHANNEL_X));
+         gen8_set_src0_da16_swiz_y(inst,
+                                   BRW_GET_SWZ(reg.dw1.bits.swizzle,
+                                               BRW_CHANNEL_Y));
+         gen8_set_src0_da16_swiz_z(inst,
+                                   BRW_GET_SWZ(reg.dw1.bits.swizzle,
+                                               BRW_CHANNEL_Z));
+         gen8_set_src0_da16_swiz_w(inst,
+                                   BRW_GET_SWZ(reg.dw1.bits.swizzle,
+                                               BRW_CHANNEL_W));
+
+         /* This is an oddity of the fact that we're using the same
+          * descriptions for registers in both Align16 and Align1 modes.
+          */
+         if (reg.vstride == BRW_VERTICAL_STRIDE_8)
+            gen8_set_src0_vert_stride(inst, BRW_VERTICAL_STRIDE_4);
+         else
+            gen8_set_src0_vert_stride(inst, reg.vstride);
+      }
+   } else {
+      /* Indirect addressing */
+      assert(gen8_access_mode(inst) == BRW_ALIGN_1);
+      if (reg.width == BRW_WIDTH_1 &&
+         gen8_exec_size(inst) == BRW_EXECUTE_1) {
+         gen8_set_src0_da1_hstride(inst, BRW_HORIZONTAL_STRIDE_0);
+         gen8_set_src0_vert_stride(inst, BRW_VERTICAL_STRIDE_0);
+      } else {
+         gen8_set_src0_da1_hstride(inst, reg.hstride);
+         gen8_set_src0_vert_stride(inst, reg.vstride);
+      }
+
+      gen8_set_src0_da1_width(inst, reg.width);
+      gen8_set_src0_ia1_subreg_nr(inst, reg.subnr);
+      gen8_set_src0_ia1_addr_imm(inst, reg.dw1.bits.indirect_offset);
+   }
+}
+
+void
+gen8_set_src1(const struct brw_context *brw,
+              struct gen8_instruction *inst,
+              struct brw_reg reg)
+{
+   gen8_convert_mrf_to_grf(&reg);
+
+   if (reg.file == BRW_GENERAL_REGISTER_FILE)
+      assert(reg.nr < BRW_MAX_GRF);
+
+   gen8_validate_reg(inst, reg);
+
+   gen8_set_src1_reg_file(inst, reg.file);
+   gen8_set_src1_reg_type(inst,
+                          brw_reg_type_to_hw_type(brw, reg.type, reg.file));
+   gen8_set_src1_abs(inst, reg.abs);
+   gen8_set_src1_negate(inst, reg.negate);
+
+   /* Only src1 can be an immediate in two-argument instructions. */
+   assert(gen8_src0_reg_file(inst) != BRW_IMMEDIATE_VALUE);
+
+   if (reg.file == BRW_IMMEDIATE_VALUE) {
+      inst->data[3] = reg.dw1.ud;
+      return;
+   }
+
+   gen8_set_src1_address_mode(inst, reg.address_mode);
+
+   if (reg.address_mode == BRW_ADDRESS_DIRECT) {
+      gen8_set_src1_da_reg_nr(inst, reg.nr);
+
+      if (gen8_access_mode(inst) == BRW_ALIGN_1) {
+         /* Set Src0.SubRegNum[4:0] */
+         gen8_set_src1_da1_subreg_nr(inst, reg.subnr);
+
+         if (reg.width == BRW_WIDTH_1 && gen8_exec_size(inst) == BRW_EXECUTE_1) {
+            gen8_set_src1_da1_hstride(inst, BRW_HORIZONTAL_STRIDE_0);
+            gen8_set_src1_vert_stride(inst, BRW_VERTICAL_STRIDE_0);
+         } else {
+            gen8_set_src1_da1_hstride(inst, reg.hstride);
+            gen8_set_src1_vert_stride(inst, reg.vstride);
+         }
+         gen8_set_src1_da1_width(inst, reg.width);
+      } else {
+         /* Align16 SubRegNum only has a single bit (bit 4; bits 3:0 MBZ). */
+         assert(reg.subnr == 0 || reg.subnr == 16);
+         gen8_set_src1_da16_subreg_nr(inst, reg.subnr >> 4);
+
+         gen8_set_src1_da16_swiz_x(inst,
+                                   BRW_GET_SWZ(reg.dw1.bits.swizzle,
+                                               BRW_CHANNEL_X));
+         gen8_set_src1_da16_swiz_y(inst,
+                                   BRW_GET_SWZ(reg.dw1.bits.swizzle,
+                                               BRW_CHANNEL_Y));
+         gen8_set_src1_da16_swiz_z(inst,
+                                   BRW_GET_SWZ(reg.dw1.bits.swizzle,
+                                               BRW_CHANNEL_Z));
+         gen8_set_src1_da16_swiz_w(inst,
+                                   BRW_GET_SWZ(reg.dw1.bits.swizzle,
+                                               BRW_CHANNEL_W));
+
+         /* This is an oddity of the fact that we're using the same
+          * descriptions for registers in both Align16 and Align1 modes.
+          */
+         if (reg.vstride == BRW_VERTICAL_STRIDE_8)
+            gen8_set_src1_vert_stride(inst, BRW_VERTICAL_STRIDE_4);
+         else
+            gen8_set_src1_vert_stride(inst, reg.vstride);
+      }
+   } else {
+      /* Indirect addressing */
+      assert(gen8_access_mode(inst) == BRW_ALIGN_1);
+      if (reg.width == BRW_WIDTH_1 && gen8_exec_size(inst) == BRW_EXECUTE_1) {
+         gen8_set_src1_da1_hstride(inst, BRW_HORIZONTAL_STRIDE_0);
+         gen8_set_src1_vert_stride(inst, BRW_VERTICAL_STRIDE_0);
+      } else {
+         gen8_set_src1_da1_hstride(inst, reg.hstride);
+         gen8_set_src1_vert_stride(inst, reg.vstride);
+      }
+
+      gen8_set_src1_da1_width(inst, reg.width);
+      gen8_set_src1_ia1_subreg_nr(inst, reg.subnr);
+      gen8_set_src1_ia1_addr_imm(inst, reg.dw1.bits.indirect_offset);
+   }
+}
+
+/**
+ * Set the Message Descriptor and Extended Message Descriptor fields
+ * for SEND messages.
+ *
+ * \note This zeroes out the Function Control bits, so it must be called
+ *       \b before filling out any message-specific data.  Callers can
+ *       choose not to fill in irrelevant bits; they will be zero.
+ */
+static void
+gen8_set_message_descriptor(const struct brw_context *brw,
+                            struct gen8_instruction *inst,
+                            enum brw_message_target sfid,
+                            unsigned msg_length,
+                            unsigned response_length,
+                            bool header_present,
+                            bool end_of_thread)
+{
+   gen8_set_src1(brw, inst, brw_imm_d(0));
+
+   gen8_set_sfid(inst, sfid);
+   gen8_set_mlen(inst, msg_length);
+   gen8_set_rlen(inst, response_length);
+   gen8_set_header_present(inst, header_present);
+   gen8_set_eot(inst, end_of_thread);
+}
+
+void
+gen8_set_urb_message(const struct brw_context *brw,
+                     struct gen8_instruction *inst,
+                     enum brw_urb_write_flags flags,
+                     unsigned msg_length,
+                     unsigned response_length,
+                     unsigned offset,
+                     bool interleave)
+{
+   gen8_set_message_descriptor(brw, inst, BRW_SFID_URB,
+                               msg_length, response_length,
+                               true, flags & BRW_URB_WRITE_EOT);
+   gen8_set_src0(brw, inst, brw_vec8_grf(GEN7_MRF_HACK_START + 1, 0));
+   if (flags & BRW_URB_WRITE_OWORD) {
+      assert(msg_length == 2);
+      gen8_set_urb_opcode(inst, BRW_URB_OPCODE_WRITE_OWORD);
+   } else {
+      gen8_set_urb_opcode(inst, BRW_URB_OPCODE_WRITE_HWORD);
+   }
+   gen8_set_urb_global_offset(inst, offset);
+   gen8_set_urb_interleave(inst, interleave);
+   gen8_set_urb_per_slot_offset(inst,
+                                flags & BRW_URB_WRITE_PER_SLOT_OFFSET ? 1 : 0);
+}
+
+void
+gen8_set_sampler_message(const struct brw_context *brw,
+                         struct gen8_instruction *inst,
+                         unsigned binding_table_index,
+                         unsigned sampler,
+                         unsigned msg_type,
+                         unsigned response_length,
+                         unsigned msg_length,
+                         bool header_present,
+                         unsigned simd_mode)
+{
+   gen8_set_message_descriptor(brw, inst, BRW_SFID_SAMPLER, msg_length,
+                               response_length, header_present, false);
+
+   gen8_set_binding_table_index(inst, binding_table_index);
+   gen8_set_sampler(inst, sampler);
+   gen8_set_sampler_msg_type(inst, msg_type);
+   gen8_set_sampler_simd_mode(inst, simd_mode);
+}
+
+void
+gen8_set_dp_message(const struct brw_context *brw,
+                    struct gen8_instruction *inst,
+                    enum brw_message_target sfid,
+                    unsigned binding_table_index,
+                    unsigned msg_type,
+                    unsigned msg_control,
+                    unsigned mlen,
+                    unsigned rlen,
+                    bool header_present,
+                    bool end_of_thread)
+{
+   gen8_set_message_descriptor(brw, inst, sfid, mlen, rlen, header_present,
+                               end_of_thread);
+   gen8_set_binding_table_index(inst, binding_table_index);
+   gen8_set_dp_message_type(inst, msg_type);
+   gen8_set_dp_message_control(inst, msg_control);
+}
diff --git a/src/mesa/drivers/dri/i965/gen8_instruction.h b/src/mesa/drivers/dri/i965/gen8_instruction.h
new file mode 100644
index 00000000000..7d8b6054966
--- /dev/null
+++ b/src/mesa/drivers/dri/i965/gen8_instruction.h
@@ -0,0 +1,399 @@
+/*
+ * Copyright © 2012 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+/**
+ * @file gen8_instruction.h
+ *
+ * A representation of a Gen8+ EU instruction, with helper methods to get
+ * and set various fields.  This is the actual hardware format.
+ */
+
+#ifndef GEN8_INSTRUCTION_H
+#define GEN8_INSTRUCTION_H
+
+#include <stdio.h>
+#include <stdint.h>
+
+#include "brw_context.h"
+#include "brw_reg.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct gen8_instruction {
+   uint32_t data[4];
+};
+
+static inline unsigned gen8_instruction_bits(struct gen8_instruction *inst,
+                                             unsigned high,
+                                             unsigned low);
+static inline void gen8_instruction_set_bits(struct gen8_instruction *inst,
+                                             unsigned high,
+                                             unsigned low,
+                                             unsigned value);
+
+#define F(name, high, low) \
+static inline void gen8_set_##name(struct gen8_instruction *inst, unsigned v) \
+{ \
+   gen8_instruction_set_bits(inst, high, low, v); \
+} \
+static inline unsigned gen8_##name(struct gen8_instruction *inst) \
+{ \
+   return gen8_instruction_bits(inst, high, low); \
+}
+
+F(src1_vert_stride,    120, 117)
+F(src1_da1_width,      116, 114)
+F(src1_da16_swiz_w,    115, 114)
+F(src1_da16_swiz_z,    113, 112)
+F(src1_da1_hstride,    113, 112)
+F(src1_address_mode,   111, 111)
+/** Src1.SrcMod @{ */
+F(src1_negate,         110, 110)
+F(src1_abs,            109, 109)
+/** @} */
+F(src1_ia1_subreg_nr,  108, 105)
+F(src1_da_reg_nr,      108, 101)
+F(src1_da16_subreg_nr, 100, 100)
+F(src1_da1_subreg_nr,  100,  96)
+F(src1_da16_swiz_y,     99,  98)
+F(src1_da16_swiz_x,     97,  96)
+F(src1_reg_type,        94,  91)
+F(src1_reg_file,        90,  89)
+F(src0_vert_stride,     88,  85)
+F(src0_da1_width,       84,  82)
+F(src0_da16_swiz_w,     83,  82)
+F(src0_da16_swiz_z,     81,  80)
+F(src0_da1_hstride,     81,  80)
+F(src0_address_mode,    79,  79)
+/** Src0.SrcMod @{ */
+F(src0_negate,          78,  78)
+F(src0_abs,             77,  77)
+/** @} */
+F(src0_ia1_subreg_nr,   76,  73)
+F(src0_da_reg_nr,       76,  69)
+F(src0_da16_subreg_nr,  68,  68)
+F(src0_da1_subreg_nr,   68,  64)
+F(src0_da16_swiz_y,     67,  66)
+F(src0_da16_swiz_x,     65,  64)
+F(dst_address_mode,     63,  63)
+F(dst_da1_hstride,      62,  61)
+F(dst_ia1_subreg_nr,    60,  57)
+F(dst_da_reg_nr,        60,  53)
+F(dst_da16_subreg_nr,   52,  52)
+F(dst_da1_subreg_nr,    52,  48)
+F(da16_writemask,       51,  48) /* Dst.ChanEn */
+F(src0_reg_type,        46,  43)
+F(src0_reg_file,        42,  41)
+F(dst_reg_type,         40,  37)
+F(dst_reg_file,         36,  35)
+F(mask_control,         34,  34)
+F(flag_reg_nr,          33,  33)
+F(flag_subreg_nr,       32,  32)
+F(saturate,             31,  31)
+F(branch_control,       30,  30)
+F(debug_control,        30,  30)
+F(cmpt_control,         29,  29)
+F(acc_wr_control,       28,  28)
+F(cond_modifier,        27,  24)
+F(exec_size,            23,  21)
+F(pred_inv,             20,  20)
+F(pred_control,         19,  16)
+F(thread_control,       15,  14)
+F(qtr_control,          13,  12)
+F(nib_control,          11,  11)
+F(no_dd_check,          10,  10)
+F(no_dd_clear,           9,   9)
+F(access_mode,           8,   8)
+/* Bit 7 is Reserved (for future Opcode expansion) */
+F(opcode,                6,   0)
+
+/**
+ * Three-source instructions:
+ *  @{
+ */
+F(src2_3src_reg_nr,    125, 118)
+F(src2_3src_subreg_nr, 117, 115)
+F(src2_3src_swizzle,   114, 107)
+F(src2_3src_rep_ctrl,  106, 106)
+F(src1_3src_reg_nr,    104,  97)
+/* src1_3src_subreg_nr spans word boundaries and has to be handled specially */
+F(src1_3src_swizzle,    93,  86)
+F(src1_3src_rep_ctrl,   85,  85)
+F(src0_3src_reg_nr,     83,  76)
+F(src0_3src_subreg_nr,  75,  73)
+F(src0_3src_swizzle,    72,  65)
+F(src0_3src_rep_ctrl,   64,  64)
+F(dst_3src_reg_nr,      63,  56)
+F(dst_3src_subreg_nr,   55,  53)
+F(dst_3src_writemask,   52,  49)
+F(dst_3src_type,        48,  46)
+F(src_3src_type,        45,  43)
+F(src2_3src_negate,     42,  42)
+F(src2_3src_abs,        41,  41)
+F(src1_3src_negate,     40,  40)
+F(src1_3src_abs,        39,  39)
+F(src0_3src_negate,     38,  38)
+F(src0_3src_abs,        37,  37)
+/** @} */
+
+/**
+ * Fields for SEND messages:
+ *  @{
+ */
+F(eot,                 127, 127)
+F(mlen,                124, 121)
+F(rlen,                120, 116)
+F(header_present,      115, 115)
+F(function_control,    114,  96)
+F(sfid,                 27,  24)
+F(math_function,        27,  24)
+/** @} */
+
+/**
+ * URB message function control bits:
+ *  @{
+ */
+F(urb_per_slot_offset, 113, 113)
+F(urb_interleave,      111, 111)
+F(urb_global_offset,   110, 100)
+F(urb_opcode,           99,  96)
+/** @} */
+
+/* Message descriptor bits */
+#define MD(name, high, low) F(name, (high + 96), (low + 96))
+
+/**
+ * Sampler message function control bits:
+ *  @{
+ */
+MD(sampler_simd_mode,   18,  17)
+MD(sampler_msg_type,    16,  12)
+MD(sampler,             11,   8)
+MD(binding_table_index,  7,   0) /* also used by other messages */
+/** @} */
+
+/**
+ * Data port message function control bits:
+ *  @{
+ */
+MD(dp_category,         18,  18)
+MD(dp_message_type,     17,  14)
+MD(dp_message_control,  13,   8)
+/** @} */
+
+/**
+ * Render Target message function control bits:
+ *  @{
+ */
+MD(rt_last,             12,  12)
+MD(rt_slot_group,       11,  11)
+MD(rt_message_type,     10,   8)
+/** @} */
+
+/**
+ * Thread Spawn message function control bits:
+ *  @{
+ */
+MD(ts_resource_select,   4,   4)
+MD(ts_request_type,      1,   1)
+MD(ts_opcode,            0,   0)
+/** @} */
+
+/**
+ * Video Motion Estimation message function control bits:
+ *  @{
+ */
+F(vme_message_type,     14,  13)
+/** @} */
+
+/**
+ * Check & Refinement Engine message function control bits:
+ *  @{
+ */
+F(cre_message_type,     14,  13)
+/** @} */
+
+#undef MD
+#undef F
+
+static inline void
+gen8_set_src1_3src_subreg_nr(struct gen8_instruction *inst, unsigned v)
+{
+   assert((v & ~0x7) == 0);
+
+   gen8_instruction_set_bits(inst, 95, 94, v & 0x3);
+   gen8_instruction_set_bits(inst, 96, 96, v >> 2);
+}
+
+static inline unsigned
+gen8_src1_3src_subreg_nr(struct gen8_instruction *inst)
+{
+   return gen8_instruction_bits(inst, 95, 94) |
+          (gen8_instruction_bits(inst, 96, 96) << 2);
+}
+
+#define GEN8_IA1_ADDR_IMM(reg, nine, high, low)                              \
+static inline void                                                           \
+gen8_set_##reg##_ia1_addr_imm(struct gen8_instruction *inst, unsigned value) \
+{                                                                            \
+   assert((value & ~0x3ff) == 0);                                            \
+   gen8_instruction_set_bits(inst, high, low, value & 0x1ff);                \
+   gen8_instruction_set_bits(inst, nine, nine, value >> 9);                  \
+}                                                                            \
+                                                                             \
+static inline unsigned                                                       \
+gen8_##reg##_ia1_addr_imm(struct gen8_instruction *inst)                     \
+{                                                                            \
+   return gen8_instruction_bits(inst, high, low) |                           \
+          (gen8_instruction_bits(inst, nine, nine) << 9);                    \
+}
+
+/* AddrImm[9:0] for Align1 Indirect Addressing */
+GEN8_IA1_ADDR_IMM(src1, 121, 104, 96)
+GEN8_IA1_ADDR_IMM(src0,  95,  72, 64)
+GEN8_IA1_ADDR_IMM(dst,   47,  56, 48)
+
+/**
+ * Flow control instruction bits:
+ *  @{
+ */
+static inline unsigned gen8_uip(struct gen8_instruction *inst)
+{
+   return inst->data[2];
+}
+static inline void gen8_set_uip(struct gen8_instruction *inst, unsigned uip)
+{
+   inst->data[2] = uip;
+}
+static inline unsigned gen8_jip(struct gen8_instruction *inst)
+{
+   return inst->data[3];
+}
+static inline void gen8_set_jip(struct gen8_instruction *inst, unsigned jip)
+{
+   inst->data[3] = jip;
+}
+/** @} */
+
+static inline int gen8_src1_imm_d(struct gen8_instruction *inst)
+{
+   return inst->data[3];
+}
+static inline unsigned gen8_src1_imm_ud(struct gen8_instruction *inst)
+{
+   return inst->data[3];
+}
+static inline float gen8_src1_imm_f(struct gen8_instruction *inst)
+{
+   fi_type ft;
+
+   ft.u = inst->data[3];
+   return ft.f;
+}
+
+void gen8_set_dst(const struct brw_context *brw,
+                  struct gen8_instruction *inst, struct brw_reg reg);
+void gen8_set_src0(const struct brw_context *brw,
+                   struct gen8_instruction *inst, struct brw_reg reg);
+void gen8_set_src1(const struct brw_context *brw,
+                   struct gen8_instruction *inst, struct brw_reg reg);
+
+void gen8_set_urb_message(const struct brw_context *brw,
+                          struct gen8_instruction *inst,
+                          enum brw_urb_write_flags flags,
+                          unsigned mlen, unsigned rlen,
+                          unsigned offset, bool interleave);
+
+void gen8_set_sampler_message(const struct brw_context *brw,
+                              struct gen8_instruction *inst,
+                              unsigned binding_table_index, unsigned sampler,
+                              unsigned msg_type, unsigned rlen, unsigned mlen,
+                              bool header_present, unsigned simd_mode);
+
+void gen8_set_dp_message(const struct brw_context *brw,
+                         struct gen8_instruction *inst,
+                         enum brw_message_target sfid,
+                         unsigned binding_table_index,
+                         unsigned msg_type,
+                         unsigned msg_control,
+                         unsigned msg_length,
+                         unsigned response_length,
+                         bool header_present,
+                         bool end_of_thread);
+
+/** Disassemble the instruction. */
+int gen8_disassemble(FILE *file, struct gen8_instruction *inst, int gen);
+
+
+/**
+ * Fetch a set of contiguous bits from the instruction.
+ *
+ * Bits indexes range from 0..127; fields may not cross 32-bit boundaries.
+ */
+static inline unsigned
+gen8_instruction_bits(struct gen8_instruction *inst, unsigned high, unsigned low)
+{
+   /* We assume the field doesn't cross 32-bit boundaries. */
+   const unsigned word = high / 32;
+   assert(word == low / 32);
+
+   high %= 32;
+   low %= 32;
+
+   const unsigned mask = (((1 << (high - low + 1)) - 1) << low);
+
+   return (inst->data[word] & mask) >> low;
+}
+
+/**
+ * Set bits in the instruction, with proper shifting and masking.
+ *
+ * Bits indexes range from 0..127; fields may not cross 32-bit boundaries.
+ */
+static inline void
+gen8_instruction_set_bits(struct gen8_instruction *inst,
+                          unsigned high,
+                          unsigned low,
+                          unsigned value)
+{
+   const unsigned word = high / 32;
+   assert(word == low / 32);
+
+   high %= 32;
+   low %= 32;
+
+   const unsigned mask = (((1 << (high - low + 1)) - 1) << low);
+
+   /* Make sure the supplied value actually fits in the given bitfield. */
+   assert((value & (mask >> low)) == value);
+
+   inst->data[word] = (inst->data[word] & ~mask) | ((value << low) & mask);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif