diff options
| -rw-r--r-- | src/mesa/drivers/dri/i965/Makefile.sources | 10 | ||||
| -rw-r--r-- | src/mesa/drivers/dri/i965/brw_context.h | 3 | ||||
| -rw-r--r-- | src/mesa/drivers/dri/i965/brw_pipe_control.c | 243 | ||||
| -rw-r--r-- | src/mesa/drivers/dri/i965/brw_pipe_control.h | 58 | ||||
| -rw-r--r-- | src/mesa/drivers/dri/i965/brw_state.h | 31 | ||||
| -rw-r--r-- | src/mesa/drivers/dri/i965/genX_pipe_control.c | 243 | ||||
| -rw-r--r-- | src/mesa/drivers/dri/i965/meson.build | 4 |
7 files changed, 362 insertions, 230 deletions
diff --git a/src/mesa/drivers/dri/i965/Makefile.sources b/src/mesa/drivers/dri/i965/Makefile.sources index e01c1e5e0de..d1c23cb2fc0 100644 --- a/src/mesa/drivers/dri/i965/Makefile.sources +++ b/src/mesa/drivers/dri/i965/Makefile.sources @@ -116,51 +116,61 @@ i965_FILES = \ i965_gen4_FILES = \ genX_blorp_exec.c \ genX_boilerplate.h \ + genX_pipe_control.c \ genX_state_upload.c i965_gen45_FILES = \ genX_blorp_exec.c \ genX_boilerplate.h \ + genX_pipe_control.c \ genX_state_upload.c i965_gen5_FILES = \ genX_blorp_exec.c \ genX_boilerplate.h \ + genX_pipe_control.c \ genX_state_upload.c i965_gen6_FILES = \ genX_blorp_exec.c \ genX_boilerplate.h \ + genX_pipe_control.c \ genX_state_upload.c i965_gen7_FILES = \ genX_blorp_exec.c \ genX_boilerplate.h \ + genX_pipe_control.c \ genX_state_upload.c i965_gen75_FILES = \ genX_blorp_exec.c \ genX_boilerplate.h \ + genX_pipe_control.c \ genX_state_upload.c i965_gen8_FILES = \ genX_blorp_exec.c \ genX_boilerplate.h \ + genX_pipe_control.c \ genX_state_upload.c i965_gen9_FILES = \ genX_blorp_exec.c \ genX_boilerplate.h \ + genX_pipe_control.c \ genX_state_upload.c i965_gen10_FILES = \ genX_blorp_exec.c \ genX_boilerplate.h \ + genX_pipe_control.c \ genX_state_upload.c i965_gen11_FILES = \ genX_blorp_exec.c \ genX_boilerplate.h \ + genX_pipe_control.c \ genX_state_upload.c i965_oa_GENERATED_FILES = \ diff --git a/src/mesa/drivers/dri/i965/brw_context.h b/src/mesa/drivers/dri/i965/brw_context.h index 301d3305133..8b46b678064 100644 --- a/src/mesa/drivers/dri/i965/brw_context.h +++ b/src/mesa/drivers/dri/i965/brw_context.h @@ -754,6 +754,9 @@ struct brw_context uint32_t report_id); void (*emit_compute_walker)(struct brw_context *brw); + void (*emit_raw_pipe_control)(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, + uint64_t imm); } vtbl; struct brw_bufmgr *bufmgr; diff --git a/src/mesa/drivers/dri/i965/brw_pipe_control.c b/src/mesa/drivers/dri/i965/brw_pipe_control.c index 704f65100d3..0e96f76a22e 100644 --- a/src/mesa/drivers/dri/i965/brw_pipe_control.c +++ b/src/mesa/drivers/dri/i965/brw_pipe_control.c @@ -23,201 +23,11 @@ #include "brw_context.h" #include "brw_defines.h" +#include "brw_state.h" #include "intel_batchbuffer.h" #include "intel_fbo.h" /** - * According to the latest documentation, any PIPE_CONTROL with the - * "Command Streamer Stall" bit set must also have another bit set, - * with five different options: - * - * - Render Target Cache Flush - * - Depth Cache Flush - * - Stall at Pixel Scoreboard - * - Post-Sync Operation - * - Depth Stall - * - DC Flush Enable - * - * I chose "Stall at Pixel Scoreboard" since we've used it effectively - * in the past, but the choice is fairly arbitrary. - */ -static void -gen8_add_cs_stall_workaround_bits(uint32_t *flags) -{ - uint32_t wa_bits = PIPE_CONTROL_RENDER_TARGET_FLUSH | - PIPE_CONTROL_DEPTH_CACHE_FLUSH | - PIPE_CONTROL_WRITE_IMMEDIATE | - PIPE_CONTROL_WRITE_DEPTH_COUNT | - PIPE_CONTROL_WRITE_TIMESTAMP | - PIPE_CONTROL_STALL_AT_SCOREBOARD | - PIPE_CONTROL_DEPTH_STALL | - PIPE_CONTROL_DATA_CACHE_FLUSH; - - /* If we're doing a CS stall, and don't already have one of the - * workaround bits set, add "Stall at Pixel Scoreboard." - */ - if ((*flags & PIPE_CONTROL_CS_STALL) != 0 && (*flags & wa_bits) == 0) - *flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD; -} - -/* Implement the WaCsStallAtEveryFourthPipecontrol workaround on IVB, BYT: - * - * "Every 4th PIPE_CONTROL command, not counting the PIPE_CONTROL with - * only read-cache-invalidate bit(s) set, must have a CS_STALL bit set." - * - * Note that the kernel does CS stalls between batches, so we only need - * to count them within a batch. - */ -static uint32_t -gen7_cs_stall_every_four_pipe_controls(struct brw_context *brw, uint32_t flags) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - if (devinfo->gen == 7 && !devinfo->is_haswell) { - if (flags & PIPE_CONTROL_CS_STALL) { - /* If we're doing a CS stall, reset the counter and carry on. */ - brw->pipe_controls_since_last_cs_stall = 0; - return 0; - } - - /* If this is the fourth pipe control without a CS stall, do one now. */ - if (++brw->pipe_controls_since_last_cs_stall == 4) { - brw->pipe_controls_since_last_cs_stall = 0; - return PIPE_CONTROL_CS_STALL; - } - } - return 0; -} - -/* #1130 from gen10 workarounds page in h/w specs: - * "Enable Depth Stall on every Post Sync Op if Render target Cache Flush is - * not enabled in same PIPE CONTROL and Enable Pixel score board stall if - * Render target cache flush is enabled." - * - * Applicable to CNL B0 and C0 steppings only. - */ -static void -gen10_add_rcpfe_workaround_bits(uint32_t *flags) -{ - if (*flags & PIPE_CONTROL_RENDER_TARGET_FLUSH) { - *flags = *flags | PIPE_CONTROL_STALL_AT_SCOREBOARD; - } else if (*flags & - (PIPE_CONTROL_WRITE_IMMEDIATE | - PIPE_CONTROL_WRITE_DEPTH_COUNT | - PIPE_CONTROL_WRITE_TIMESTAMP)) { - *flags = *flags | PIPE_CONTROL_DEPTH_STALL; - } -} - -static void -brw_emit_pipe_control(struct brw_context *brw, uint32_t flags, - struct brw_bo *bo, uint32_t offset, uint64_t imm) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - if (devinfo->gen >= 8) { - if (devinfo->gen == 8) - gen8_add_cs_stall_workaround_bits(&flags); - - if (flags & PIPE_CONTROL_VF_CACHE_INVALIDATE) { - if (devinfo->gen == 9) { - /* The PIPE_CONTROL "VF Cache Invalidation Enable" bit description - * lists several workarounds: - * - * "Project: SKL, KBL, BXT - * - * If the VF Cache Invalidation Enable is set to a 1 in a - * PIPE_CONTROL, a separate Null PIPE_CONTROL, all bitfields - * sets to 0, with the VF Cache Invalidation Enable set to 0 - * needs to be sent prior to the PIPE_CONTROL with VF Cache - * Invalidation Enable set to a 1." - */ - brw_emit_pipe_control_flush(brw, 0); - } - - if (devinfo->gen >= 9) { - /* THE PIPE_CONTROL "VF Cache Invalidation Enable" docs continue: - * - * "Project: BDW+ - * - * When VF Cache Invalidate is set “Post Sync Operation” must - * be enabled to “Write Immediate Data” or “Write PS Depth - * Count” or “Write Timestamp”." - * - * If there's a BO, we're already doing some kind of write. - * If not, add a write to the workaround BO. - * - * XXX: This causes GPU hangs on Broadwell, so restrict it to - * Gen9+ for now...see this bug for more information: - * https://bugs.freedesktop.org/show_bug.cgi?id=103787 - */ - if (!bo) { - flags |= PIPE_CONTROL_WRITE_IMMEDIATE; - bo = brw->workaround_bo; - } - } - } - - if (devinfo->gen == 10) - gen10_add_rcpfe_workaround_bits(&flags); - - BEGIN_BATCH(6); - OUT_BATCH(_3DSTATE_PIPE_CONTROL | (6 - 2)); - OUT_BATCH(flags); - if (bo) { - OUT_RELOC64(bo, RELOC_WRITE, offset); - } else { - OUT_BATCH(0); - OUT_BATCH(0); - } - OUT_BATCH(imm); - OUT_BATCH(imm >> 32); - ADVANCE_BATCH(); - } else if (devinfo->gen >= 6) { - if (devinfo->gen == 6 && - (flags & PIPE_CONTROL_RENDER_TARGET_FLUSH)) { - /* Hardware workaround: SNB B-Spec says: - * - * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush - * Enable = 1, a PIPE_CONTROL with any non-zero post-sync-op is - * required. - */ - brw_emit_post_sync_nonzero_flush(brw); - } - - flags |= gen7_cs_stall_every_four_pipe_controls(brw, flags); - - /* PPGTT/GGTT is selected by DW2 bit 2 on Sandybridge, but DW1 bit 24 - * on later platforms. We always use PPGTT on Gen7+. - */ - unsigned gen6_gtt = devinfo->gen == 6 ? PIPE_CONTROL_GLOBAL_GTT_WRITE : 0; - - BEGIN_BATCH(5); - OUT_BATCH(_3DSTATE_PIPE_CONTROL | (5 - 2)); - OUT_BATCH(flags); - if (bo) { - OUT_RELOC(bo, RELOC_WRITE | RELOC_NEEDS_GGTT, gen6_gtt | offset); - } else { - OUT_BATCH(0); - } - OUT_BATCH(imm); - OUT_BATCH(imm >> 32); - ADVANCE_BATCH(); - } else { - BEGIN_BATCH(4); - OUT_BATCH(_3DSTATE_PIPE_CONTROL | flags | (4 - 2)); - if (bo) { - OUT_RELOC(bo, RELOC_WRITE, PIPE_CONTROL_GLOBAL_GTT_WRITE | offset); - } else { - OUT_BATCH(0); - } - OUT_BATCH(imm); - OUT_BATCH(imm >> 32); - ADVANCE_BATCH(); - } -} - -/** * Emit a PIPE_CONTROL with various flushing flags. * * The caller is responsible for deciding what flags are appropriate for the @@ -246,7 +56,7 @@ brw_emit_pipe_control_flush(struct brw_context *brw, uint32_t flags) flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL); } - brw_emit_pipe_control(brw, flags, NULL, 0, 0); + brw->vtbl.emit_raw_pipe_control(brw, flags, NULL, 0, 0); } /** @@ -262,7 +72,7 @@ brw_emit_pipe_control_write(struct brw_context *brw, uint32_t flags, struct brw_bo *bo, uint32_t offset, uint64_t imm) { - brw_emit_pipe_control(brw, flags, bo, offset, imm); + brw->vtbl.emit_raw_pipe_control(brw, flags, bo, offset, imm); } /** @@ -357,14 +167,14 @@ gen7_emit_vs_workaround_flush(struct brw_context *brw) void gen10_emit_isp_disable(struct brw_context *brw) { - brw_emit_pipe_control(brw, - PIPE_CONTROL_STALL_AT_SCOREBOARD | - PIPE_CONTROL_CS_STALL, - NULL, 0, 0); - brw_emit_pipe_control(brw, - PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE | - PIPE_CONTROL_CS_STALL, - NULL, 0, 0); + brw->vtbl.emit_raw_pipe_control(brw, + PIPE_CONTROL_STALL_AT_SCOREBOARD | + PIPE_CONTROL_CS_STALL, + NULL, 0, 0); + brw->vtbl.emit_raw_pipe_control(brw, + PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE | + PIPE_CONTROL_CS_STALL, + NULL, 0, 0); brw->vs.base.push_constants_dirty = true; brw->tcs.base.push_constants_dirty = true; @@ -561,6 +371,37 @@ int brw_init_pipe_control(struct brw_context *brw, const struct gen_device_info *devinfo) { + switch (devinfo->gen) { + case 11: + brw->vtbl.emit_raw_pipe_control = gen11_emit_raw_pipe_control; + break; + case 10: + brw->vtbl.emit_raw_pipe_control = gen10_emit_raw_pipe_control; + break; + case 9: + brw->vtbl.emit_raw_pipe_control = gen9_emit_raw_pipe_control; + break; + case 8: + brw->vtbl.emit_raw_pipe_control = gen8_emit_raw_pipe_control; + break; + case 7: + brw->vtbl.emit_raw_pipe_control = + devinfo->is_haswell ? gen75_emit_raw_pipe_control + : gen7_emit_raw_pipe_control; + break; + case 6: + brw->vtbl.emit_raw_pipe_control = gen6_emit_raw_pipe_control; + break; + case 5: + brw->vtbl.emit_raw_pipe_control = gen5_emit_raw_pipe_control; + break; + case 4: + brw->vtbl.emit_raw_pipe_control = + devinfo->is_g4x ? gen45_emit_raw_pipe_control + : gen4_emit_raw_pipe_control; + break; + } + if (devinfo->gen < 6) return 0; diff --git a/src/mesa/drivers/dri/i965/brw_pipe_control.h b/src/mesa/drivers/dri/i965/brw_pipe_control.h index 69b1c7c31e6..e213f43a4f7 100644 --- a/src/mesa/drivers/dri/i965/brw_pipe_control.h +++ b/src/mesa/drivers/dri/i965/brw_pipe_control.h @@ -32,34 +32,38 @@ struct brw_bo; * * PIPE_CONTROL operation, a combination MI_FLUSH and register write with * additional flushing control. + * + * The bits here are not the actual hardware values. The actual values + * shift around a bit per-generation, so we just have flags for each + * potential operation, and use genxml to encode the actual packet. */ -#define _3DSTATE_PIPE_CONTROL (CMD_3D | (3 << 27) | (2 << 24)) -#define PIPE_CONTROL_LRI_WRITE_IMMEDIATE (1 << 23) /* Gen7+ */ -#define PIPE_CONTROL_CS_STALL (1 << 20) -#define PIPE_CONTROL_GLOBAL_SNAPSHOT_COUNT_RESET (1 << 19) -#define PIPE_CONTROL_TLB_INVALIDATE (1 << 18) -#define PIPE_CONTROL_SYNC_GFDT (1 << 17) -#define PIPE_CONTROL_MEDIA_STATE_CLEAR (1 << 16) -#define PIPE_CONTROL_NO_WRITE (0 << 14) -#define PIPE_CONTROL_WRITE_IMMEDIATE (1 << 14) -#define PIPE_CONTROL_WRITE_DEPTH_COUNT (2 << 14) -#define PIPE_CONTROL_WRITE_TIMESTAMP (3 << 14) -#define PIPE_CONTROL_DEPTH_STALL (1 << 13) -#define PIPE_CONTROL_RENDER_TARGET_FLUSH (1 << 12) -#define PIPE_CONTROL_INSTRUCTION_INVALIDATE (1 << 11) -#define PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE (1 << 10) /* GM45+ only */ -#define PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE (1 << 9) -#define PIPE_CONTROL_INTERRUPT_ENABLE (1 << 8) -#define PIPE_CONTROL_FLUSH_ENABLE (1 << 7) /* Gen7+ only */ -/* GT */ -#define PIPE_CONTROL_DATA_CACHE_FLUSH (1 << 5) -#define PIPE_CONTROL_VF_CACHE_INVALIDATE (1 << 4) -#define PIPE_CONTROL_CONST_CACHE_INVALIDATE (1 << 3) -#define PIPE_CONTROL_STATE_CACHE_INVALIDATE (1 << 2) -#define PIPE_CONTROL_STALL_AT_SCOREBOARD (1 << 1) -#define PIPE_CONTROL_DEPTH_CACHE_FLUSH (1 << 0) -#define PIPE_CONTROL_PPGTT_WRITE (0 << 2) -#define PIPE_CONTROL_GLOBAL_GTT_WRITE (1 << 2) +enum pipe_control_flags +{ + PIPE_CONTROL_FLUSH_LLC = (1 << 1), + PIPE_CONTROL_LRI_POST_SYNC_OP = (1 << 2), + PIPE_CONTROL_STORE_DATA_INDEX = (1 << 3), + PIPE_CONTROL_CS_STALL = (1 << 4), + PIPE_CONTROL_GLOBAL_SNAPSHOT_COUNT_RESET = (1 << 5), + PIPE_CONTROL_SYNC_GFDT = (1 << 6), + PIPE_CONTROL_TLB_INVALIDATE = (1 << 7), + PIPE_CONTROL_MEDIA_STATE_CLEAR = (1 << 8), + PIPE_CONTROL_WRITE_IMMEDIATE = (1 << 9), + PIPE_CONTROL_WRITE_DEPTH_COUNT = (1 << 10), + PIPE_CONTROL_WRITE_TIMESTAMP = (1 << 11), + PIPE_CONTROL_DEPTH_STALL = (1 << 12), + PIPE_CONTROL_RENDER_TARGET_FLUSH = (1 << 13), + PIPE_CONTROL_INSTRUCTION_INVALIDATE = (1 << 14), + PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE = (1 << 15), + PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE = (1 << 16), + PIPE_CONTROL_NOTIFY_ENABLE = (1 << 17), + PIPE_CONTROL_FLUSH_ENABLE = (1 << 18), + PIPE_CONTROL_DATA_CACHE_FLUSH = (1 << 19), + PIPE_CONTROL_VF_CACHE_INVALIDATE = (1 << 20), + PIPE_CONTROL_CONST_CACHE_INVALIDATE = (1 << 21), + PIPE_CONTROL_STATE_CACHE_INVALIDATE = (1 << 22), + PIPE_CONTROL_STALL_AT_SCOREBOARD = (1 << 23), + PIPE_CONTROL_DEPTH_CACHE_FLUSH = (1 << 24), +}; #define PIPE_CONTROL_CACHE_FLUSH_BITS \ (PIPE_CONTROL_DEPTH_CACHE_FLUSH | PIPE_CONTROL_DATA_CACHE_FLUSH | \ diff --git a/src/mesa/drivers/dri/i965/brw_state.h b/src/mesa/drivers/dri/i965/brw_state.h index 546d103d1a4..402ae692f9f 100644 --- a/src/mesa/drivers/dri/i965/brw_state.h +++ b/src/mesa/drivers/dri/i965/brw_state.h @@ -95,6 +95,37 @@ extern const struct brw_tracked_state gen7_urb; extern const struct brw_tracked_state gen8_pma_fix; extern const struct brw_tracked_state brw_cs_work_groups_surface; +void gen4_emit_raw_pipe_control(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, + uint64_t imm); +void gen45_emit_raw_pipe_control(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, + uint64_t imm); +void gen5_emit_raw_pipe_control(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, + uint64_t imm); +void gen6_emit_raw_pipe_control(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, + uint64_t imm); +void gen7_emit_raw_pipe_control(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, + uint64_t imm); +void gen75_emit_raw_pipe_control(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, + uint64_t imm); +void gen8_emit_raw_pipe_control(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, + uint64_t imm); +void gen9_emit_raw_pipe_control(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, + uint64_t imm); +void gen10_emit_raw_pipe_control(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, + uint64_t imm); +void gen11_emit_raw_pipe_control(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, + uint64_t imm); + static inline bool brw_state_dirty(const struct brw_context *brw, GLuint mesa_flags, uint64_t brw_flags) diff --git a/src/mesa/drivers/dri/i965/genX_pipe_control.c b/src/mesa/drivers/dri/i965/genX_pipe_control.c new file mode 100644 index 00000000000..8eb37444253 --- /dev/null +++ b/src/mesa/drivers/dri/i965/genX_pipe_control.c @@ -0,0 +1,243 @@ +/* + * Copyright © 2017 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "genX_boilerplate.h" +#include "brw_defines.h" +#include "brw_state.h" + +/** + * According to the latest documentation, any PIPE_CONTROL with the + * "Command Streamer Stall" bit set must also have another bit set, + * with five different options: + * + * - Render Target Cache Flush + * - Depth Cache Flush + * - Stall at Pixel Scoreboard + * - Post-Sync Operation + * - Depth Stall + * - DC Flush Enable + * + * I chose "Stall at Pixel Scoreboard" since we've used it effectively + * in the past, but the choice is fairly arbitrary. + */ +static void +gen8_add_cs_stall_workaround_bits(uint32_t *flags) +{ + uint32_t wa_bits = PIPE_CONTROL_RENDER_TARGET_FLUSH | + PIPE_CONTROL_DEPTH_CACHE_FLUSH | + PIPE_CONTROL_WRITE_IMMEDIATE | + PIPE_CONTROL_WRITE_DEPTH_COUNT | + PIPE_CONTROL_WRITE_TIMESTAMP | + PIPE_CONTROL_STALL_AT_SCOREBOARD | + PIPE_CONTROL_DEPTH_STALL | + PIPE_CONTROL_DATA_CACHE_FLUSH; + + /* If we're doing a CS stall, and don't already have one of the + * workaround bits set, add "Stall at Pixel Scoreboard." + */ + if ((*flags & PIPE_CONTROL_CS_STALL) != 0 && (*flags & wa_bits) == 0) + *flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD; +} + +/* Implement the WaCsStallAtEveryFourthPipecontrol workaround on IVB, BYT: + * + * "Every 4th PIPE_CONTROL command, not counting the PIPE_CONTROL with + * only read-cache-invalidate bit(s) set, must have a CS_STALL bit set." + * + * Note that the kernel does CS stalls between batches, so we only need + * to count them within a batch. + */ +static uint32_t +gen7_cs_stall_every_four_pipe_controls(struct brw_context *brw, uint32_t flags) +{ + if (GEN_GEN == 7 && !GEN_IS_HASWELL) { + if (flags & PIPE_CONTROL_CS_STALL) { + /* If we're doing a CS stall, reset the counter and carry on. */ + brw->pipe_controls_since_last_cs_stall = 0; + return 0; + } + + /* If this is the fourth pipe control without a CS stall, do one now. */ + if (++brw->pipe_controls_since_last_cs_stall == 4) { + brw->pipe_controls_since_last_cs_stall = 0; + return PIPE_CONTROL_CS_STALL; + } + } + return 0; +} + +/* #1130 from gen10 workarounds page in h/w specs: + * "Enable Depth Stall on every Post Sync Op if Render target Cache Flush is + * not enabled in same PIPE CONTROL and Enable Pixel score board stall if + * Render target cache flush is enabled." + * + * Applicable to CNL B0 and C0 steppings only. + */ +static void +gen10_add_rcpfe_workaround_bits(uint32_t *flags) +{ + if (*flags & PIPE_CONTROL_RENDER_TARGET_FLUSH) { + *flags = *flags | PIPE_CONTROL_STALL_AT_SCOREBOARD; + } else if (*flags & + (PIPE_CONTROL_WRITE_IMMEDIATE | + PIPE_CONTROL_WRITE_DEPTH_COUNT | + PIPE_CONTROL_WRITE_TIMESTAMP)) { + *flags = *flags | PIPE_CONTROL_DEPTH_STALL; + } +} + +static unsigned +flags_to_post_sync_op(uint32_t flags) +{ + flags &= PIPE_CONTROL_WRITE_IMMEDIATE | + PIPE_CONTROL_WRITE_DEPTH_COUNT | + PIPE_CONTROL_WRITE_TIMESTAMP; + + assert(util_bitcount(flags) <= 1); + + if (flags & PIPE_CONTROL_WRITE_IMMEDIATE) + return WriteImmediateData; + + if (flags & PIPE_CONTROL_WRITE_DEPTH_COUNT) + return WritePSDepthCount; + + if (flags & PIPE_CONTROL_WRITE_TIMESTAMP) + return WriteTimestamp; + + return 0; +} + +void +genX(emit_raw_pipe_control)(struct brw_context *brw, uint32_t flags, + struct brw_bo *bo, uint32_t offset, uint64_t imm) +{ + if (GEN_GEN >= 8) { + if (GEN_GEN == 8) + gen8_add_cs_stall_workaround_bits(&flags); + + if (flags & PIPE_CONTROL_VF_CACHE_INVALIDATE) { + if (GEN_GEN == 9) { + /* The PIPE_CONTROL "VF Cache Invalidation Enable" bit description + * lists several workarounds: + * + * "Project: SKL, KBL, BXT + * + * If the VF Cache Invalidation Enable is set to a 1 in a + * PIPE_CONTROL, a separate Null PIPE_CONTROL, all bitfields + * sets to 0, with the VF Cache Invalidation Enable set to 0 + * needs to be sent prior to the PIPE_CONTROL with VF Cache + * Invalidation Enable set to a 1." + */ + brw_emit_pipe_control_flush(brw, 0); + } + + if (GEN_GEN >= 9) { + /* THE PIPE_CONTROL "VF Cache Invalidation Enable" docs continue: + * + * "Project: BDW+ + * + * When VF Cache Invalidate is set “Post Sync Operation” must + * be enabled to “Write Immediate Data” or “Write PS Depth + * Count” or “Write Timestamp”." + * + * If there's a BO, we're already doing some kind of write. + * If not, add a write to the workaround BO. + * + * XXX: This causes GPU hangs on Broadwell, so restrict it to + * Gen9+ for now...see this bug for more information: + * https://bugs.freedesktop.org/show_bug.cgi?id=103787 + */ + if (!bo) { + flags |= PIPE_CONTROL_WRITE_IMMEDIATE; + bo = brw->workaround_bo; + } + } + } + + if (GEN_GEN == 10) + gen10_add_rcpfe_workaround_bits(&flags); + } else if (GEN_GEN >= 6) { + if (GEN_GEN == 6 && + (flags & PIPE_CONTROL_RENDER_TARGET_FLUSH)) { + /* Hardware workaround: SNB B-Spec says: + * + * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush + * Enable = 1, a PIPE_CONTROL with any non-zero post-sync-op is + * required. + */ + brw_emit_post_sync_nonzero_flush(brw); + } + + flags |= gen7_cs_stall_every_four_pipe_controls(brw, flags); + } + + brw_batch_emit(brw, GENX(PIPE_CONTROL), pc) { + #if GEN_GEN >= 9 + pc.FlushLLC = 0; + #endif + #if GEN_GEN >= 7 + pc.LRIPostSyncOperation = NoLRIOperation; + pc.PipeControlFlushEnable = flags & PIPE_CONTROL_FLUSH_ENABLE; + pc.DCFlushEnable = flags & PIPE_CONTROL_DATA_CACHE_FLUSH; + #endif + #if GEN_GEN >= 6 + pc.StoreDataIndex = 0; + pc.CommandStreamerStallEnable = flags & PIPE_CONTROL_CS_STALL; + pc.GlobalSnapshotCountReset = + flags & PIPE_CONTROL_GLOBAL_SNAPSHOT_COUNT_RESET; + pc.TLBInvalidate = flags & PIPE_CONTROL_TLB_INVALIDATE; + pc.GenericMediaStateClear = flags & PIPE_CONTROL_MEDIA_STATE_CLEAR; + pc.StallAtPixelScoreboard = flags & PIPE_CONTROL_STALL_AT_SCOREBOARD; + pc.RenderTargetCacheFlushEnable = + flags & PIPE_CONTROL_RENDER_TARGET_FLUSH; + pc.DepthCacheFlushEnable = flags & PIPE_CONTROL_DEPTH_CACHE_FLUSH; + pc.StateCacheInvalidationEnable = + flags & PIPE_CONTROL_STATE_CACHE_INVALIDATE; + pc.VFCacheInvalidationEnable = flags & PIPE_CONTROL_VF_CACHE_INVALIDATE; + pc.ConstantCacheInvalidationEnable = + flags & PIPE_CONTROL_CONST_CACHE_INVALIDATE; + #else + pc.WriteCacheFlush = flags & PIPE_CONTROL_RENDER_TARGET_FLUSH; + #endif + pc.PostSyncOperation = flags_to_post_sync_op(flags); + pc.DepthStallEnable = flags & PIPE_CONTROL_DEPTH_STALL; + pc.InstructionCacheInvalidateEnable = + flags & PIPE_CONTROL_INSTRUCTION_INVALIDATE; + pc.NotifyEnable = flags & PIPE_CONTROL_NOTIFY_ENABLE; + #if GEN_GEN >= 5 || GEN_IS_G4X + pc.IndirectStatePointersDisable = + flags & PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE; + #endif + #if GEN_GEN >= 6 + pc.TextureCacheInvalidationEnable = + flags & PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; + #elif GEN_GEN == 5 || GEN_IS_G4X + pc.TextureCacheFlushEnable = + flags & PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; + #endif + pc.Address = ggtt_bo(bo, offset); + if (GEN_GEN < 7 && bo) + pc.DestinationAddressType = DAT_GGTT; + pc.ImmediateData = imm; + } +} diff --git a/src/mesa/drivers/dri/i965/meson.build b/src/mesa/drivers/dri/i965/meson.build index abb7b241f78..95f8cd21cba 100644 --- a/src/mesa/drivers/dri/i965/meson.build +++ b/src/mesa/drivers/dri/i965/meson.build @@ -137,8 +137,8 @@ i965_gen_libs = [] foreach v : ['40', '45', '50', '60', '70', '75', '80', '90', '100', '110'] i965_gen_libs += static_library( 'i965_gen@0@'.format(v), - ['genX_blorp_exec.c', 'genX_boilerplate.h', 'genX_state_upload.c', - gen_xml_pack], + ['genX_blorp_exec.c', 'genX_boilerplate.h', 'genX_pipe_control.c', + 'genX_state_upload.c', gen_xml_pack], include_directories : [inc_common, inc_intel, inc_dri_common], c_args : [ c_vis_args, no_override_init_args, c_sse2_args, |