i965: Update SOL state for Broadwell.

Unlike on Gen7, we can directly set the offset via the state packet.
We also -have- to: the kernel SOL reset code won't work anymore.

v2: Fix copy and paste mistake in buffer stride setup; drop stale
    comment (caught by Eric Anholt).  Add a perf_debug for missing
    MOCS setup.

v3: Rebase on Paul Berry's changes to CurrentVertexProgram.

v4: Fix SO Write Offset handling.  We need to set bits 20 and 21 so the
    hardware both loads and saves the offset.  There's also a
    restriction that 3DSTATE_SO_BUFFER can only be programmed once per
    buffer between primitives, so the "reset to zero" code needed
    reworking.  Fixes most of the transform feedback Piglit tests.

Signed-off-by: Kenneth Graunke <[email protected]>
Reviewed-by: Eric Anholt <[email protected]> [v2]

7 files changed, 215 insertions, 19 deletions

diff --git a/src/mesa/drivers/dri/i965/Makefile.sources b/src/mesa/drivers/dri/i965/Makefile.sources
index 1617b707503..7e280458177 100644
--- a/src/mesa/drivers/dri/i965/Makefile.sources
+++ b/src/mesa/drivers/dri/i965/Makefile.sources
@@ -150,6 +150,7 @@ i965_FILES = \
 	gen8_instruction.c \
 	gen8_misc_state.c \
 	gen8_sf_state.c \
+	gen8_sol_state.c \
 	gen8_vec4_generator.cpp \
 	gen8_vs_state.c \
 	gen8_wm_depth_stencil.c \
diff --git a/src/mesa/drivers/dri/i965/brw_context.h b/src/mesa/drivers/dri/i965/brw_context.h
index b6a1363a6a1..d63d6ba2f44 100644
--- a/src/mesa/drivers/dri/i965/brw_context.h
+++ b/src/mesa/drivers/dri/i965/brw_context.h
@@ -898,6 +898,9 @@ struct brw_transform_feedback_object {
    /** A buffer to hold SO_WRITE_OFFSET(n) values while paused. */
    drm_intel_bo *offset_bo;
 
+   /** If true, SO_WRITE_OFFSET(n) should be reset to zero at next use. */
+   bool zero_offsets;
+
    /** The most recent primitive mode (GL_TRIANGLES/GL_POINTS/GL_LINES). */
    GLenum primitive_mode;
 
diff --git a/src/mesa/drivers/dri/i965/brw_defines.h b/src/mesa/drivers/dri/i965/brw_defines.h
index d21a3643126..d8b6af47ed5 100644
--- a/src/mesa/drivers/dri/i965/brw_defines.h
+++ b/src/mesa/drivers/dri/i965/brw_defines.h
@@ -2022,8 +2022,11 @@ enum brw_wm_barycentric_interp_mode {
 
 #define _3DSTATE_SO_BUFFER                    0x7918 /* GEN7+ */
 /* DW1 */
+# define GEN8_SO_BUFFER_ENABLE                          (1 << 31)
 # define SO_BUFFER_INDEX_SHIFT				29
 # define SO_BUFFER_INDEX_MASK				INTEL_MASK(30, 29)
+# define GEN8_SO_BUFFER_OFFSET_WRITE_ENABLE             (1 << 21)
+# define GEN8_SO_BUFFER_OFFSET_ADDRESS_ENABLE           (1 << 20)
 # define SO_BUFFER_PITCH_SHIFT				0
 # define SO_BUFFER_PITCH_MASK				INTEL_MASK(11, 0)
 /* DW2: start address */
diff --git a/src/mesa/drivers/dri/i965/brw_state.h b/src/mesa/drivers/dri/i965/brw_state.h
index 3bffabd4c16..141061980c2 100644
--- a/src/mesa/drivers/dri/i965/brw_state.h
+++ b/src/mesa/drivers/dri/i965/brw_state.h
@@ -142,6 +142,7 @@ extern const struct brw_tracked_state gen8_raster_state;
 extern const struct brw_tracked_state gen8_sbe_state;
 extern const struct brw_tracked_state gen8_sf_state;
 extern const struct brw_tracked_state gen8_state_base_address;
+extern const struct brw_tracked_state gen8_sol_state;
 extern const struct brw_tracked_state gen8_vertices;
 extern const struct brw_tracked_state gen8_vf_topology;
 extern const struct brw_tracked_state gen8_vs_state;
diff --git a/src/mesa/drivers/dri/i965/brw_state_upload.c b/src/mesa/drivers/dri/i965/brw_state_upload.c
index 3c5af09f24f..70240af2bad 100644
--- a/src/mesa/drivers/dri/i965/brw_state_upload.c
+++ b/src/mesa/drivers/dri/i965/brw_state_upload.c
@@ -299,7 +299,7 @@ static const struct brw_tracked_state *gen8_atoms[] =
    &gen8_disable_stages,
    &gen8_vs_state,
    &gen7_gs_state,
-   &gen7_sol_state,
+   &gen8_sol_state,
    &gen6_clip_state,
    &gen8_raster_state,
    &gen8_sbe_state,
diff --git a/src/mesa/drivers/dri/i965/gen7_sol_state.c b/src/mesa/drivers/dri/i965/gen7_sol_state.c
index 5d2f243eba1..1b58efbf829 100644
--- a/src/mesa/drivers/dri/i965/gen7_sol_state.c
+++ b/src/mesa/drivers/dri/i965/gen7_sol_state.c
@@ -409,8 +409,13 @@ gen7_begin_transform_feedback(struct gl_context *ctx, GLenum mode,
    struct brw_transform_feedback_object *brw_obj =
       (struct brw_transform_feedback_object *) obj;
 
-   intel_batchbuffer_flush(brw);
-   brw->batch.needs_sol_reset = true;
+   /* Reset the SO buffer offsets to 0. */
+   if (brw->gen >= 8) {
+      brw_obj->zero_offsets = true;
+   } else {
+      intel_batchbuffer_flush(brw);
+      brw->batch.needs_sol_reset = true;
+   }
 
    /* We're about to lose the information needed to compute the number of
     * vertices written during the last Begin/EndTransformFeedback section,
@@ -466,14 +471,16 @@ gen7_pause_transform_feedback(struct gl_context *ctx,
    intel_batchbuffer_emit_mi_flush(brw);
 
    /* Save the SOL buffer offset register values. */
-   for (int i = 0; i < 4; i++) {
-      BEGIN_BATCH(3);
-      OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
-      OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
-      OUT_RELOC(brw_obj->offset_bo,
-                I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
-                i * sizeof(uint32_t));
-      ADVANCE_BATCH();
+   if (brw->gen < 8) {
+      for (int i = 0; i < 4; i++) {
+         BEGIN_BATCH(3);
+         OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
+         OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
+         OUT_RELOC(brw_obj->offset_bo,
+                   I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
+                   i * sizeof(uint32_t));
+         ADVANCE_BATCH();
+      }
    }
 
    /* Store the temporary ending value of the SO_NUM_PRIMS_WRITTEN counters.
@@ -493,14 +500,16 @@ gen7_resume_transform_feedback(struct gl_context *ctx,
       (struct brw_transform_feedback_object *) obj;
 
    /* Reload the SOL buffer offset registers. */
-   for (int i = 0; i < 4; i++) {
-      BEGIN_BATCH(3);
-      OUT_BATCH(GEN7_MI_LOAD_REGISTER_MEM | (3 - 2));
-      OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
-      OUT_RELOC(brw_obj->offset_bo,
-                I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
-                i * sizeof(uint32_t));
-      ADVANCE_BATCH();
+   if (brw->gen < 8) {
+      for (int i = 0; i < 4; i++) {
+         BEGIN_BATCH(3);
+         OUT_BATCH(GEN7_MI_LOAD_REGISTER_MEM | (3 - 2));
+         OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
+         OUT_RELOC(brw_obj->offset_bo,
+                   I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
+                   i * sizeof(uint32_t));
+         ADVANCE_BATCH();
+      }
    }
 
    /* Store the new starting value of the SO_NUM_PRIMS_WRITTEN counters. */
diff --git a/src/mesa/drivers/dri/i965/gen8_sol_state.c b/src/mesa/drivers/dri/i965/gen8_sol_state.c
new file mode 100644
index 00000000000..35a77ac37cf
--- /dev/null
+++ b/src/mesa/drivers/dri/i965/gen8_sol_state.c
@@ -0,0 +1,179 @@
+/*
+ * Copyright © 2012 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+/**
+ * @file gen8_sol_state.c
+ *
+ * Controls the stream output logic (SOL) stage of the gen8 hardware, which is
+ * used to implement GL_EXT_transform_feedback.
+ */
+
+#include "brw_context.h"
+#include "brw_state.h"
+#include "brw_defines.h"
+#include "intel_batchbuffer.h"
+#include "intel_buffer_objects.h"
+#include "main/transformfeedback.h"
+
+static void
+gen8_upload_3dstate_so_buffers(struct brw_context *brw)
+{
+   struct gl_context *ctx = &brw->ctx;
+   /* BRW_NEW_TRANSFORM_FEEDBACK */
+   struct gl_transform_feedback_object *xfb_obj =
+      ctx->TransformFeedback.CurrentObject;
+   struct brw_transform_feedback_object *brw_obj =
+      (struct brw_transform_feedback_object *) xfb_obj;
+
+   /* Set up the up to 4 output buffers.  These are the ranges defined in the
+    * gl_transform_feedback_object.
+    */
+   for (int i = 0; i < 4; i++) {
+      struct intel_buffer_object *bufferobj =
+         intel_buffer_object(xfb_obj->Buffers[i]);
+
+      if (!bufferobj) {
+         BEGIN_BATCH(8);
+         OUT_BATCH(_3DSTATE_SO_BUFFER << 16 | (8 - 2));
+         OUT_BATCH((i << SO_BUFFER_INDEX_SHIFT));
+         OUT_BATCH(0);
+         OUT_BATCH(0);
+         OUT_BATCH(0);
+         OUT_BATCH(0);
+         OUT_BATCH(0);
+         OUT_BATCH(0);
+         ADVANCE_BATCH();
+         continue;
+      }
+
+      uint32_t start = xfb_obj->Offset[i];
+      assert(start % 4 == 0);
+      uint32_t end = ALIGN(start + xfb_obj->Size[i], 4);
+      drm_intel_bo *bo =
+         intel_bufferobj_buffer(brw, bufferobj, start, end - start);
+      assert(end <= bo->size);
+
+      perf_debug("Missing MOCS setup for 3DSTATE_SO_BUFFER.");
+
+      BEGIN_BATCH(8);
+      OUT_BATCH(_3DSTATE_SO_BUFFER << 16 | (8 - 2));
+      OUT_BATCH(GEN8_SO_BUFFER_ENABLE | (i << SO_BUFFER_INDEX_SHIFT) |
+                GEN8_SO_BUFFER_OFFSET_WRITE_ENABLE |
+                GEN8_SO_BUFFER_OFFSET_ADDRESS_ENABLE);
+      OUT_RELOC64(bo, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, start);
+      OUT_BATCH(xfb_obj->Size[i] / 4 - 1);
+      OUT_RELOC64(brw_obj->offset_bo,
+                  I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
+                  i * sizeof(uint32_t));
+      if (brw_obj->zero_offsets)
+         OUT_BATCH(0); /* Zero out the offset and write that to offset_bo */
+      else
+         OUT_BATCH(0xFFFFFFFF); /* Use offset_bo as the "Stream Offset." */
+      ADVANCE_BATCH();
+   }
+   brw_obj->zero_offsets = false;
+}
+
+static void
+gen8_upload_3dstate_streamout(struct brw_context *brw, bool active,
+                              struct brw_vue_map *vue_map)
+{
+   struct gl_context *ctx = &brw->ctx;
+
+   /* BRW_NEW_VERTEX_PROGRAM */
+   const struct gl_shader_program *vs_prog =
+      ctx->Shader.CurrentProgram[MESA_SHADER_VERTEX];
+   /* BRW_NEW_TRANSFORM_FEEDBACK */
+   const struct gl_transform_feedback_info *linked_xfb_info =
+      &vs_prog->LinkedTransformFeedback;
+   struct gl_transform_feedback_object *xfb_obj =
+      ctx->TransformFeedback.CurrentObject;
+   uint32_t dw1 = 0, dw2 = 0, dw3 = 0, dw4 = 0;
+
+   if (active) {
+      int urb_entry_read_offset = 0;
+      int urb_entry_read_length = (vue_map->num_slots + 1) / 2 -
+         urb_entry_read_offset;
+
+      dw1 |= SO_FUNCTION_ENABLE;
+      dw1 |= SO_STATISTICS_ENABLE;
+
+      /* _NEW_LIGHT */
+      if (ctx->Light.ProvokingVertex != GL_FIRST_VERTEX_CONVENTION)
+         dw1 |= SO_REORDER_TRAILING;
+
+      /* We always read the whole vertex.  This could be reduced at some
+       * point by reading less and offsetting the register index in the
+       * SO_DECLs.
+       */
+      dw2 |= urb_entry_read_offset << SO_STREAM_0_VERTEX_READ_OFFSET_SHIFT;
+      dw2 |= (urb_entry_read_length - 1) << SO_STREAM_0_VERTEX_READ_LENGTH_SHIFT;
+
+      /* Set buffer pitches; 0 means unbound. */
+      if (xfb_obj->Buffers[0])
+         dw3 |= linked_xfb_info->BufferStride[0] * 4;
+      if (xfb_obj->Buffers[1])
+         dw3 |= (linked_xfb_info->BufferStride[1] * 4) << 16;
+      if (xfb_obj->Buffers[2])
+         dw4 |= linked_xfb_info->BufferStride[2] * 4;
+      if (xfb_obj->Buffers[3])
+         dw4 |= (linked_xfb_info->BufferStride[3] * 4) << 16;
+   }
+
+   BEGIN_BATCH(5);
+   OUT_BATCH(_3DSTATE_STREAMOUT << 16 | (5 - 2));
+   OUT_BATCH(dw1);
+   OUT_BATCH(dw2);
+   OUT_BATCH(dw3);
+   OUT_BATCH(dw4);
+   ADVANCE_BATCH();
+}
+
+static void
+upload_sol_state(struct brw_context *brw)
+{
+   struct gl_context *ctx = &brw->ctx;
+   /* BRW_NEW_TRANSFORM_FEEDBACK */
+   bool active = _mesa_is_xfb_active_and_unpaused(ctx);
+
+   if (active) {
+      gen8_upload_3dstate_so_buffers(brw);
+      /* BRW_NEW_VUE_MAP_GEOM_OUT */
+      gen7_upload_3dstate_so_decl_list(brw, &brw->vue_map_geom_out);
+   }
+
+   gen8_upload_3dstate_streamout(brw, active, &brw->vue_map_geom_out);
+}
+
+const struct brw_tracked_state gen8_sol_state = {
+   .dirty = {
+      .mesa  = _NEW_LIGHT,
+      .brw   = BRW_NEW_BATCH |
+               BRW_NEW_RASTERIZER_DISCARD |
+               BRW_NEW_TRANSFORM_FEEDBACK |
+               BRW_NEW_VERTEX_PROGRAM |
+               BRW_NEW_VUE_MAP_GEOM_OUT,
+      .cache = 0,
+   },
+   .emit = upload_sol_state,
+};