aboutsummaryrefslogtreecommitdiffstats
path: root/src/gallium/drivers/iris/iris_pipe_control.c
blob: 4adaf14ff999f596c077ffe0e8b59cfe6bb39927 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
/*
 * Copyright © 2017 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

/**
 * @file iris_pipe_control.c
 *
 * PIPE_CONTROL is the main flushing and synchronization primitive on Intel
 * GPUs.  It can invalidate caches, stall until rendering reaches various
 * stages of completion, write to memory, and other things.  In a way, it's
 * a swiss army knife command - it has all kinds of capabilities, but some
 * significant limitations as well.
 *
 * Unfortunately, it's notoriously complicated and difficult to use.  Many
 * sub-commands can't be used together.  Some are meant to be used at the
 * top of the pipeline (invalidating caches before drawing), while some are
 * meant to be used at the end (stalling or flushing after drawing).
 *
 * Also, there's a list of restrictions a mile long, which vary by generation.
 * Do this before doing that, or suffer the consequences (usually a GPU hang).
 *
 * This file contains helpers for emitting them safely.  You can simply call
 * iris_emit_pipe_control_flush() with the desired operations (as logical
 * PIPE_CONTROL_* bits), and it will take care of splitting it into multiple
 * PIPE_CONTROL commands as necessary.  The per-generation workarounds are
 * applied in iris_emit_raw_pipe_control() in iris_state.c.
 */

#include "iris_context.h"
#include "util/hash_table.h"
#include "util/set.h"

/**
 * Emit a PIPE_CONTROL with various flushing flags.
 *
 * The caller is responsible for deciding what flags are appropriate for the
 * given generation.
 */
void
iris_emit_pipe_control_flush(struct iris_batch *batch,
                             const char *reason,
                             uint32_t flags)
{
   if ((flags & PIPE_CONTROL_CACHE_FLUSH_BITS) &&
       (flags & PIPE_CONTROL_CACHE_INVALIDATE_BITS)) {
      /* A pipe control command with flush and invalidate bits set
       * simultaneously is an inherently racy operation on Gen6+ if the
       * contents of the flushed caches were intended to become visible from
       * any of the invalidated caches.  Split it in two PIPE_CONTROLs, the
       * first one should stall the pipeline to make sure that the flushed R/W
       * caches are coherent with memory once the specified R/O caches are
       * invalidated.  On pre-Gen6 hardware the (implicit) R/O cache
       * invalidation seems to happen at the bottom of the pipeline together
       * with any write cache flush, so this shouldn't be a concern.  In order
       * to ensure a full stall, we do an end-of-pipe sync.
       */
      iris_emit_end_of_pipe_sync(batch, reason,
                                 flags & PIPE_CONTROL_CACHE_FLUSH_BITS);
      flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL);
   }

   batch->screen->vtbl.emit_raw_pipe_control(batch, reason, flags, NULL, 0, 0);
}

/**
 * Emit a PIPE_CONTROL that writes to a buffer object.
 *
 * \p flags should contain one of the following items:
 *  - PIPE_CONTROL_WRITE_IMMEDIATE
 *  - PIPE_CONTROL_WRITE_TIMESTAMP
 *  - PIPE_CONTROL_WRITE_DEPTH_COUNT
 */
void
iris_emit_pipe_control_write(struct iris_batch *batch,
                             const char *reason, uint32_t flags,
                             struct iris_bo *bo, uint32_t offset,
                             uint64_t imm)
{
   batch->screen->vtbl.emit_raw_pipe_control(batch, reason, flags, bo, offset, imm);
}

/*
 * From Sandybridge PRM, volume 2, "1.7.2 End-of-Pipe Synchronization":
 *
 *  Write synchronization is a special case of end-of-pipe
 *  synchronization that requires that the render cache and/or depth
 *  related caches are flushed to memory, where the data will become
 *  globally visible. This type of synchronization is required prior to
 *  SW (CPU) actually reading the result data from memory, or initiating
 *  an operation that will use as a read surface (such as a texture
 *  surface) a previous render target and/or depth/stencil buffer
 *
 * From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
 *
 *  Exercising the write cache flush bits (Render Target Cache Flush
 *  Enable, Depth Cache Flush Enable, DC Flush) in PIPE_CONTROL only
 *  ensures the write caches are flushed and doesn't guarantee the data
 *  is globally visible.
 *
 *  SW can track the completion of the end-of-pipe-synchronization by
 *  using "Notify Enable" and "PostSync Operation - Write Immediate
 *  Data" in the PIPE_CONTROL command.
 */
void
iris_emit_end_of_pipe_sync(struct iris_batch *batch,
                           const char *reason, uint32_t flags)
{
   /* From Sandybridge PRM, volume 2, "1.7.3.1 Writing a Value to Memory":
    *
    *    "The most common action to perform upon reaching a synchronization
    *    point is to write a value out to memory. An immediate value
    *    (included with the synchronization command) may be written."
    *
    * From Broadwell PRM, volume 7, "End-of-Pipe Synchronization":
    *
    *    "In case the data flushed out by the render engine is to be read
    *    back in to the render engine in coherent manner, then the render
    *    engine has to wait for the fence completion before accessing the
    *    flushed data. This can be achieved by following means on various
    *    products: PIPE_CONTROL command with CS Stall and the required
    *    write caches flushed with Post-Sync-Operation as Write Immediate
    *    Data.
    *
    *    Example:
    *       - Workload-1 (3D/GPGPU/MEDIA)
    *       - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write Immediate
    *         Data, Required Write Cache Flush bits set)
    *       - Workload-2 (Can use the data produce or output by Workload-1)
    */
   iris_emit_pipe_control_write(batch, reason,
                                flags | PIPE_CONTROL_CS_STALL |
                                PIPE_CONTROL_WRITE_IMMEDIATE,
                                batch->screen->workaround_address.bo,
                                batch->screen->workaround_address.offset, 0);
}

/**
 * Emits appropriate flushes and invalidations for any previous memory
 * operations on \p bo to be strictly ordered relative to any subsequent
 * memory operations performed from the caching domain \p access.
 *
 * This is useful because the GPU has separate incoherent caches for the
 * render target, sampler, etc., which need to be explicitly invalidated or
 * flushed in order to obtain the expected memory ordering in cases where the
 * same surface is accessed through multiple caches (e.g. due to
 * render-to-texture).
 *
 * This provides the expected memory ordering guarantees whether or not the
 * previous access was performed from the same batch or a different one, but
 * only the former case needs to be handled explicitly here, since the kernel
 * already inserts implicit flushes and synchronization in order to guarantee
 * that any data dependencies between batches are satisfied.
 *
 * Even though no flushing nor invalidation is required in order to account
 * for concurrent updates from other batches, we provide the guarantee that a
 * required synchronization operation due to a previous batch-local update
 * will never be omitted due to the influence of another thread accessing the
 * same buffer concurrently from the same caching domain: Such a concurrent
 * update will only ever change the seqno of the last update to a value
 * greater than the local value (see iris_bo_bump_seqno()), which means that
 * we will always emit at least as much flushing and invalidation as we would
 * have for the local seqno (see the coherent_seqnos comparisons below).
 */
void
iris_emit_buffer_barrier_for(struct iris_batch *batch,
                             struct iris_bo *bo,
                             enum iris_domain access)
{
   const uint32_t all_flush_bits = (PIPE_CONTROL_CACHE_FLUSH_BITS |
                                    PIPE_CONTROL_STALL_AT_SCOREBOARD |
                                    PIPE_CONTROL_FLUSH_ENABLE);
   const uint32_t flush_bits[NUM_IRIS_DOMAINS] = {
      [IRIS_DOMAIN_RENDER_WRITE] = PIPE_CONTROL_RENDER_TARGET_FLUSH,
      [IRIS_DOMAIN_DEPTH_WRITE] = PIPE_CONTROL_DEPTH_CACHE_FLUSH,
      [IRIS_DOMAIN_OTHER_WRITE] = PIPE_CONTROL_FLUSH_ENABLE,
      [IRIS_DOMAIN_OTHER_READ] = PIPE_CONTROL_STALL_AT_SCOREBOARD,
   };
   const uint32_t invalidate_bits[NUM_IRIS_DOMAINS] = {
      [IRIS_DOMAIN_RENDER_WRITE] = PIPE_CONTROL_RENDER_TARGET_FLUSH,
      [IRIS_DOMAIN_DEPTH_WRITE] = PIPE_CONTROL_DEPTH_CACHE_FLUSH,
      [IRIS_DOMAIN_OTHER_WRITE] = PIPE_CONTROL_FLUSH_ENABLE,
      [IRIS_DOMAIN_OTHER_READ] = (PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
                                  PIPE_CONTROL_CONST_CACHE_INVALIDATE),
   };
   uint32_t bits = 0;

   /* Iterate over all read/write domains first in order to handle RaW
    * and WaW dependencies, which might involve flushing the domain of
    * the previous access and invalidating the specified domain.
    */
   for (unsigned i = 0; i < IRIS_DOMAIN_OTHER_WRITE; i++) {
      assert(!iris_domain_is_read_only(i));
      if (i != access) {
         const uint64_t seqno = READ_ONCE(bo->last_seqnos[i]);

         /* Invalidate unless the most recent read/write access from
          * this domain is already guaranteed to be visible to the
          * specified domain.  Flush if the most recent access from
          * this domain occurred after its most recent flush.
          */
         if (seqno > batch->coherent_seqnos[access][i]) {
            bits |= invalidate_bits[access];

            if (seqno > batch->coherent_seqnos[i][i])
               bits |= flush_bits[i];
         }
      }
   }

   /* All read-only domains can be considered mutually coherent since
    * the order of read-only memory operations is immaterial.  If the
    * specified domain is read/write we need to iterate over them too,
    * in order to handle any WaR dependencies.
    */
   if (!iris_domain_is_read_only(access)) {
      for (unsigned i = IRIS_DOMAIN_OTHER_READ; i < NUM_IRIS_DOMAINS; i++) {
         assert(iris_domain_is_read_only(i));
         const uint64_t seqno = READ_ONCE(bo->last_seqnos[i]);

         /* Flush if the most recent access from this domain occurred
          * after its most recent flush.
          */
         if (seqno > batch->coherent_seqnos[i][i])
            bits |= flush_bits[i];
      }
   }

   /* The IRIS_DOMAIN_OTHER_WRITE kitchen-sink domain cannot be
    * considered coherent with itself since it's really a collection
    * of multiple incoherent read/write domains, so we special-case it
    * here.
    */
   const unsigned i = IRIS_DOMAIN_OTHER_WRITE;
   const uint64_t seqno = READ_ONCE(bo->last_seqnos[i]);

   /* Invalidate unless the most recent read/write access from this
    * domain is already guaranteed to be visible to the specified
    * domain.  Flush if the most recent access from this domain
    * occurred after its most recent flush.
    */
   if (seqno > batch->coherent_seqnos[access][i]) {
      bits |= invalidate_bits[access];

      if (seqno > batch->coherent_seqnos[i][i])
         bits |= flush_bits[i];
   }

   if (bits) {
      /* Stall-at-scoreboard is not expected to work in combination with other
       * flush bits.
       */
      if (bits & PIPE_CONTROL_CACHE_FLUSH_BITS)
         bits &= ~PIPE_CONTROL_STALL_AT_SCOREBOARD;

      /* Emit any required flushes and invalidations. */
      if (bits & all_flush_bits)
         iris_emit_end_of_pipe_sync(batch, "cache tracker: flush",
                                    bits & all_flush_bits);

      if (bits & ~all_flush_bits)
         iris_emit_pipe_control_flush(batch, "cache tracker: invalidate",
                                      bits & ~all_flush_bits);
   }
}

/**
 * Flush and invalidate all caches (for debugging purposes).
 */
void
iris_flush_all_caches(struct iris_batch *batch)
{
   iris_emit_pipe_control_flush(batch, "debug: flush all caches",
                                PIPE_CONTROL_CS_STALL |
                                PIPE_CONTROL_DATA_CACHE_FLUSH |
                                PIPE_CONTROL_DEPTH_CACHE_FLUSH |
                                PIPE_CONTROL_RENDER_TARGET_FLUSH |
                                PIPE_CONTROL_VF_CACHE_INVALIDATE |
                                PIPE_CONTROL_INSTRUCTION_INVALIDATE |
                                PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
                                PIPE_CONTROL_CONST_CACHE_INVALIDATE |
                                PIPE_CONTROL_STATE_CACHE_INVALIDATE);
}

static void
iris_texture_barrier(struct pipe_context *ctx, unsigned flags)
{
   struct iris_context *ice = (void *) ctx;
   struct iris_batch *render_batch = &ice->batches[IRIS_BATCH_RENDER];
   struct iris_batch *compute_batch = &ice->batches[IRIS_BATCH_COMPUTE];

   if (render_batch->contains_draw) {
      iris_batch_maybe_flush(render_batch, 48);
      iris_emit_pipe_control_flush(render_batch,
                                   "API: texture barrier (1/2)",
                                   PIPE_CONTROL_DEPTH_CACHE_FLUSH |
                                   PIPE_CONTROL_RENDER_TARGET_FLUSH |
                                   PIPE_CONTROL_CS_STALL);
      iris_emit_pipe_control_flush(render_batch,
                                   "API: texture barrier (2/2)",
                                   PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
   }

   if (compute_batch->contains_draw) {
      iris_batch_maybe_flush(compute_batch, 48);
      iris_emit_pipe_control_flush(compute_batch,
                                   "API: texture barrier (1/2)",
                                   PIPE_CONTROL_CS_STALL);
      iris_emit_pipe_control_flush(compute_batch,
                                   "API: texture barrier (2/2)",
                                   PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
   }
}

static void
iris_memory_barrier(struct pipe_context *ctx, unsigned flags)
{
   struct iris_context *ice = (void *) ctx;
   unsigned bits = PIPE_CONTROL_DATA_CACHE_FLUSH | PIPE_CONTROL_CS_STALL;

   if (flags & (PIPE_BARRIER_VERTEX_BUFFER |
                PIPE_BARRIER_INDEX_BUFFER |
                PIPE_BARRIER_INDIRECT_BUFFER)) {
      bits |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
   }

   if (flags & PIPE_BARRIER_CONSTANT_BUFFER) {
      bits |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
              PIPE_CONTROL_CONST_CACHE_INVALIDATE;
   }

   if (flags & (PIPE_BARRIER_TEXTURE | PIPE_BARRIER_FRAMEBUFFER)) {
      bits |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
              PIPE_CONTROL_RENDER_TARGET_FLUSH;
   }

   for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
      if (ice->batches[i].contains_draw) {
         iris_batch_maybe_flush(&ice->batches[i], 24);
         iris_emit_pipe_control_flush(&ice->batches[i], "API: memory barrier",
                                      bits);
      }
   }
}

void
iris_init_flush_functions(struct pipe_context *ctx)
{
   ctx->memory_barrier = iris_memory_barrier;
   ctx->texture_barrier = iris_texture_barrier;
}