/* * Copyright © 2017 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ /** * @file iris_query.c * * Query object support. This allows measuring various simple statistics * via counters on the GPU. */ #include #include #include "pipe/p_defines.h" #include "pipe/p_state.h" #include "pipe/p_context.h" #include "pipe/p_screen.h" #include "util/fast_idiv_by_const.h" #include "util/u_inlines.h" #include "util/u_upload_mgr.h" #include "iris_context.h" #include "iris_defines.h" #include "iris_fence.h" #include "iris_resource.h" #include "iris_screen.h" #include "vulkan/util/vk_util.h" #define IA_VERTICES_COUNT 0x2310 #define IA_PRIMITIVES_COUNT 0x2318 #define VS_INVOCATION_COUNT 0x2320 #define HS_INVOCATION_COUNT 0x2300 #define DS_INVOCATION_COUNT 0x2308 #define GS_INVOCATION_COUNT 0x2328 #define GS_PRIMITIVES_COUNT 0x2330 #define CL_INVOCATION_COUNT 0x2338 #define CL_PRIMITIVES_COUNT 0x2340 #define PS_INVOCATION_COUNT 0x2348 #define CS_INVOCATION_COUNT 0x2290 #define PS_DEPTH_COUNT 0x2350 #define SO_PRIM_STORAGE_NEEDED(n) (0x5240 + (n) * 8) #define SO_NUM_PRIMS_WRITTEN(n) (0x5200 + (n) * 8) #define MI_MATH (0x1a << 23) #define MI_ALU_LOAD 0x080 #define MI_ALU_LOADINV 0x480 #define MI_ALU_LOAD0 0x081 #define MI_ALU_LOAD1 0x481 #define MI_ALU_ADD 0x100 #define MI_ALU_SUB 0x101 #define MI_ALU_AND 0x102 #define MI_ALU_OR 0x103 #define MI_ALU_XOR 0x104 #define MI_ALU_STORE 0x180 #define MI_ALU_STOREINV 0x580 #define MI_ALU_R0 0x00 #define MI_ALU_R1 0x01 #define MI_ALU_R2 0x02 #define MI_ALU_R3 0x03 #define MI_ALU_R4 0x04 #define MI_ALU_SRCA 0x20 #define MI_ALU_SRCB 0x21 #define MI_ALU_ACCU 0x31 #define MI_ALU_ZF 0x32 #define MI_ALU_CF 0x33 #define _MI_ALU(op, x, y) (((op) << 20) | ((x) << 10) | (y)) #define _MI_ALU0(op) _MI_ALU(MI_ALU_##op, 0, 0) #define _MI_ALU1(op, x) _MI_ALU(MI_ALU_##op, x, 0) #define _MI_ALU2(op, x, y) _MI_ALU(MI_ALU_##op, x, y) #define MI_ALU0(op) _MI_ALU0(op) #define MI_ALU1(op, x) _MI_ALU1(op, MI_ALU_##x) #define MI_ALU2(op, x, y) _MI_ALU2(op, MI_ALU_##x, MI_ALU_##y) #define emit_lri32 ice->vtbl.load_register_imm32 #define emit_lri64 ice->vtbl.load_register_imm64 #define emit_lrr32 ice->vtbl.load_register_reg32 struct iris_query { enum pipe_query_type type; int index; bool ready; bool stalled; uint64_t result; struct iris_state_ref query_state_ref; struct iris_query_snapshots *map; struct iris_syncpt *syncpt; int batch_idx; }; struct iris_query_snapshots { /** iris_render_condition's saved MI_PREDICATE_RESULT value. */ uint64_t predicate_result; /** Have the start/end snapshots landed? */ uint64_t snapshots_landed; /** Starting and ending counter snapshots */ uint64_t start; uint64_t end; }; struct iris_query_so_overflow { uint64_t predicate_result; uint64_t snapshots_landed; struct { uint64_t prim_storage_needed[2]; uint64_t num_prims[2]; } stream[4]; }; /** * Is this type of query written by PIPE_CONTROL? */ static bool iris_is_query_pipelined(struct iris_query *q) { switch (q->type) { case PIPE_QUERY_OCCLUSION_COUNTER: case PIPE_QUERY_OCCLUSION_PREDICATE: case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: case PIPE_QUERY_TIMESTAMP: case PIPE_QUERY_TIMESTAMP_DISJOINT: case PIPE_QUERY_TIME_ELAPSED: return true; default: return false; } } static void mark_available(struct iris_context *ice, struct iris_query *q) { struct iris_batch *batch = &ice->batches[q->batch_idx]; unsigned flags = PIPE_CONTROL_WRITE_IMMEDIATE; unsigned offset = offsetof(struct iris_query_snapshots, snapshots_landed); struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res); offset += q->query_state_ref.offset; if (!iris_is_query_pipelined(q)) { ice->vtbl.store_data_imm64(batch, bo, offset, true); } else { /* Order available *after* the query results. */ flags |= PIPE_CONTROL_FLUSH_ENABLE; iris_emit_pipe_control_write(batch, flags, bo, offset, true); } } /** * Write PS_DEPTH_COUNT to q->(dest) via a PIPE_CONTROL. */ static void iris_pipelined_write(struct iris_batch *batch, struct iris_query *q, enum pipe_control_flags flags, unsigned offset) { const struct gen_device_info *devinfo = &batch->screen->devinfo; const unsigned optional_cs_stall = devinfo->gen == 9 && devinfo->gt == 4 ? PIPE_CONTROL_CS_STALL : 0; struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res); iris_emit_pipe_control_write(batch, flags | optional_cs_stall, bo, offset, 0ull); } static void write_value(struct iris_context *ice, struct iris_query *q, unsigned offset) { struct iris_batch *batch = &ice->batches[q->batch_idx]; const struct gen_device_info *devinfo = &batch->screen->devinfo; struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res); if (!iris_is_query_pipelined(q)) { iris_emit_pipe_control_flush(batch, PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD); q->stalled = true; } switch (q->type) { case PIPE_QUERY_OCCLUSION_COUNTER: case PIPE_QUERY_OCCLUSION_PREDICATE: case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: if (devinfo->gen >= 10) { /* "Driver must program PIPE_CONTROL with only Depth Stall Enable * bit set prior to programming a PIPE_CONTROL with Write PS Depth * Count sync operation." */ iris_emit_pipe_control_flush(batch, PIPE_CONTROL_DEPTH_STALL); } iris_pipelined_write(&ice->batches[IRIS_BATCH_RENDER], q, PIPE_CONTROL_WRITE_DEPTH_COUNT | PIPE_CONTROL_DEPTH_STALL, offset); break; case PIPE_QUERY_TIME_ELAPSED: case PIPE_QUERY_TIMESTAMP: case PIPE_QUERY_TIMESTAMP_DISJOINT: iris_pipelined_write(&ice->batches[IRIS_BATCH_RENDER], q, PIPE_CONTROL_WRITE_TIMESTAMP, offset); break; case PIPE_QUERY_PRIMITIVES_GENERATED: ice->vtbl.store_register_mem64(batch, q->index == 0 ? CL_INVOCATION_COUNT : SO_PRIM_STORAGE_NEEDED(q->index), bo, offset, false); break; case PIPE_QUERY_PRIMITIVES_EMITTED: ice->vtbl.store_register_mem64(batch, SO_NUM_PRIMS_WRITTEN(q->index), bo, offset, false); break; case PIPE_QUERY_PIPELINE_STATISTICS_SINGLE: { static const uint32_t index_to_reg[] = { IA_VERTICES_COUNT, IA_PRIMITIVES_COUNT, VS_INVOCATION_COUNT, GS_INVOCATION_COUNT, GS_PRIMITIVES_COUNT, CL_INVOCATION_COUNT, CL_PRIMITIVES_COUNT, PS_INVOCATION_COUNT, HS_INVOCATION_COUNT, DS_INVOCATION_COUNT, CS_INVOCATION_COUNT, }; const uint32_t reg = index_to_reg[q->index]; ice->vtbl.store_register_mem64(batch, reg, bo, offset, false); break; } default: assert(false); } } static void write_overflow_values(struct iris_context *ice, struct iris_query *q, bool end) { struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; uint32_t count = q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE ? 1 : 4; struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res); uint32_t offset = q->query_state_ref.offset; iris_emit_pipe_control_flush(batch, PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD); for (uint32_t i = 0; i < count; i++) { int s = q->index + i; int g_idx = offset + offsetof(struct iris_query_so_overflow, stream[s].num_prims[end]); int w_idx = offset + offsetof(struct iris_query_so_overflow, stream[s].prim_storage_needed[end]); ice->vtbl.store_register_mem64(batch, SO_NUM_PRIMS_WRITTEN(s), bo, g_idx, false); ice->vtbl.store_register_mem64(batch, SO_PRIM_STORAGE_NEEDED(s), bo, w_idx, false); } } uint64_t iris_timebase_scale(const struct gen_device_info *devinfo, uint64_t gpu_timestamp) { return (1000000000ull * gpu_timestamp) / devinfo->timestamp_frequency; } static uint64_t iris_raw_timestamp_delta(uint64_t time0, uint64_t time1) { if (time0 > time1) { return (1ULL << TIMESTAMP_BITS) + time1 - time0; } else { return time1 - time0; } } static bool stream_overflowed(struct iris_query_so_overflow *so, int s) { return (so->stream[s].prim_storage_needed[1] - so->stream[s].prim_storage_needed[0]) != (so->stream[s].num_prims[1] - so->stream[s].num_prims[0]); } static void calculate_result_on_cpu(const struct gen_device_info *devinfo, struct iris_query *q) { switch (q->type) { case PIPE_QUERY_OCCLUSION_PREDICATE: case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: q->result = q->map->end != q->map->start; break; case PIPE_QUERY_TIMESTAMP: case PIPE_QUERY_TIMESTAMP_DISJOINT: /* The timestamp is the single starting snapshot. */ q->result = iris_timebase_scale(devinfo, q->map->start); q->result &= (1ull << TIMESTAMP_BITS) - 1; break; case PIPE_QUERY_TIME_ELAPSED: q->result = iris_raw_timestamp_delta(q->map->start, q->map->end); q->result = iris_timebase_scale(devinfo, q->result); q->result &= (1ull << TIMESTAMP_BITS) - 1; break; case PIPE_QUERY_SO_OVERFLOW_PREDICATE: q->result = stream_overflowed((void *) q->map, q->index); break; case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE: q->result = false; for (int i = 0; i < MAX_VERTEX_STREAMS; i++) q->result |= stream_overflowed((void *) q->map, i); break; case PIPE_QUERY_PIPELINE_STATISTICS_SINGLE: q->result = q->map->end - q->map->start; /* WaDividePSInvocationCountBy4:HSW,BDW */ if (devinfo->gen == 8 && q->index == PIPE_STAT_QUERY_PS_INVOCATIONS) q->result /= 4; break; case PIPE_QUERY_OCCLUSION_COUNTER: case PIPE_QUERY_PRIMITIVES_GENERATED: case PIPE_QUERY_PRIMITIVES_EMITTED: default: q->result = q->map->end - q->map->start; break; } q->ready = true; } static void emit_alu_add(struct iris_batch *batch, unsigned dst_reg, unsigned reg_a, unsigned reg_b) { uint32_t *math = iris_get_command_space(batch, 5 * sizeof(uint32_t)); math[0] = MI_MATH | (5 - 2); math[1] = _MI_ALU2(LOAD, MI_ALU_SRCA, reg_a); math[2] = _MI_ALU2(LOAD, MI_ALU_SRCB, reg_b); math[3] = _MI_ALU0(ADD); math[4] = _MI_ALU2(STORE, dst_reg, MI_ALU_ACCU); } static void emit_alu_shl(struct iris_batch *batch, unsigned dst_reg, unsigned src_reg, unsigned shift) { assert(shift > 0); int dwords = 1 + 4 * shift; uint32_t *math = iris_get_command_space(batch, sizeof(uint32_t) * dwords); math[0] = MI_MATH | ((1 + 4 * shift) - 2); for (unsigned i = 0; i < shift; i++) { unsigned add_src = (i == 0) ? src_reg : dst_reg; math[1 + (i * 4) + 0] = _MI_ALU2(LOAD, MI_ALU_SRCA, add_src); math[1 + (i * 4) + 1] = _MI_ALU2(LOAD, MI_ALU_SRCB, add_src); math[1 + (i * 4) + 2] = _MI_ALU0(ADD); math[1 + (i * 4) + 3] = _MI_ALU2(STORE, dst_reg, MI_ALU_ACCU); } } /* Emit dwords to multiply GPR0 by N */ static void build_alu_multiply_gpr0(uint32_t *dw, unsigned *dw_count, uint32_t N) { VK_OUTARRAY_MAKE(out, dw, dw_count); #define APPEND_ALU(op, x, y) \ vk_outarray_append(&out, alu_dw) *alu_dw = _MI_ALU(MI_ALU_##op, x, y) assert(N > 0); unsigned top_bit = 31 - __builtin_clz(N); for (int i = top_bit - 1; i >= 0; i--) { /* We get our initial data in GPR0 and we write the final data out to * GPR0 but we use GPR1 as our scratch register. */ unsigned src_reg = i == top_bit - 1 ? MI_ALU_R0 : MI_ALU_R1; unsigned dst_reg = i == 0 ? MI_ALU_R0 : MI_ALU_R1; /* Shift the current value left by 1 */ APPEND_ALU(LOAD, MI_ALU_SRCA, src_reg); APPEND_ALU(LOAD, MI_ALU_SRCB, src_reg); APPEND_ALU(ADD, 0, 0); if (N & (1 << i)) { /* Store ACCU to R1 and add R0 to R1 */ APPEND_ALU(STORE, MI_ALU_R1, MI_ALU_ACCU); APPEND_ALU(LOAD, MI_ALU_SRCA, MI_ALU_R0); APPEND_ALU(LOAD, MI_ALU_SRCB, MI_ALU_R1); APPEND_ALU(ADD, 0, 0); } APPEND_ALU(STORE, dst_reg, MI_ALU_ACCU); } #undef APPEND_ALU } static void emit_mul_gpr0(struct iris_batch *batch, uint32_t N) { uint32_t num_dwords; build_alu_multiply_gpr0(NULL, &num_dwords, N); uint32_t *math = iris_get_command_space(batch, 4 * num_dwords); math[0] = MI_MATH | (num_dwords - 2); build_alu_multiply_gpr0(&math[1], &num_dwords, N); } void iris_math_div32_gpr0(struct iris_context *ice, struct iris_batch *batch, uint32_t D) { /* Zero out the top of GPR0 */ emit_lri32(batch, CS_GPR(0) + 4, 0); if (D == 0) { /* This invalid, but we should do something so we set GPR0 to 0. */ emit_lri32(batch, CS_GPR(0), 0); } else if (util_is_power_of_two_or_zero(D)) { unsigned log2_D = util_logbase2(D); assert(log2_D < 32); /* We right-shift by log2(D) by left-shifting by 32 - log2(D) and taking * the top 32 bits of the result. */ emit_alu_shl(batch, MI_ALU_R0, MI_ALU_R0, 32 - log2_D); emit_lrr32(batch, CS_GPR(0) + 0, CS_GPR(0) + 4); emit_lri32(batch, CS_GPR(0) + 4, 0); } else { struct util_fast_udiv_info m = util_compute_fast_udiv_info(D, 32, 32); assert(m.multiplier <= UINT32_MAX); if (m.pre_shift) { /* We right-shift by L by left-shifting by 32 - l and taking the top * 32 bits of the result. */ if (m.pre_shift < 32) emit_alu_shl(batch, MI_ALU_R0, MI_ALU_R0, 32 - m.pre_shift); emit_lrr32(batch, CS_GPR(0) + 0, CS_GPR(0) + 4); emit_lri32(batch, CS_GPR(0) + 4, 0); } /* Do the 32x32 multiply into gpr0 */ emit_mul_gpr0(batch, m.multiplier); if (m.increment) { /* If we need to increment, save off a copy of GPR0 */ emit_lri32(batch, CS_GPR(1) + 0, m.multiplier); emit_lri32(batch, CS_GPR(1) + 4, 0); emit_alu_add(batch, MI_ALU_R0, MI_ALU_R0, MI_ALU_R1); } /* Shift by 32 */ emit_lrr32(batch, CS_GPR(0) + 0, CS_GPR(0) + 4); emit_lri32(batch, CS_GPR(0) + 4, 0); if (m.post_shift) { /* We right-shift by L by left-shifting by 32 - l and taking the top * 32 bits of the result. */ if (m.post_shift < 32) emit_alu_shl(batch, MI_ALU_R0, MI_ALU_R0, 32 - m.post_shift); emit_lrr32(batch, CS_GPR(0) + 0, CS_GPR(0) + 4); emit_lri32(batch, CS_GPR(0) + 4, 0); } } } /* * GPR0 = (GPR0 == 0) ? 0 : 1; */ static void gpr0_to_bool(struct iris_context *ice) { struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; ice->vtbl.load_register_imm64(batch, CS_GPR(1), 1ull); static const uint32_t math[] = { MI_MATH | (9 - 2), MI_ALU2(LOAD, SRCA, R0), MI_ALU1(LOAD0, SRCB), MI_ALU0(ADD), MI_ALU2(STOREINV, R0, ZF), MI_ALU2(LOAD, SRCA, R0), MI_ALU2(LOAD, SRCB, R1), MI_ALU0(AND), MI_ALU2(STORE, R0, ACCU), }; iris_batch_emit(batch, math, sizeof(math)); } static void load_overflow_data_to_cs_gprs(struct iris_context *ice, struct iris_query *q, int idx) { struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res); uint32_t offset = q->query_state_ref.offset; ice->vtbl.load_register_mem64(batch, CS_GPR(1), bo, offset + offsetof(struct iris_query_so_overflow, stream[idx].prim_storage_needed[0])); ice->vtbl.load_register_mem64(batch, CS_GPR(2), bo, offset + offsetof(struct iris_query_so_overflow, stream[idx].prim_storage_needed[1])); ice->vtbl.load_register_mem64(batch, CS_GPR(3), bo, offset + offsetof(struct iris_query_so_overflow, stream[idx].num_prims[0])); ice->vtbl.load_register_mem64(batch, CS_GPR(4), bo, offset + offsetof(struct iris_query_so_overflow, stream[idx].num_prims[1])); } /* * R3 = R4 - R3; * R1 = R2 - R1; * R1 = R3 - R1; * R0 = R0 | R1; */ static void calc_overflow_for_stream(struct iris_context *ice) { struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; static const uint32_t maths[] = { MI_MATH | (17 - 2), MI_ALU2(LOAD, SRCA, R4), MI_ALU2(LOAD, SRCB, R3), MI_ALU0(SUB), MI_ALU2(STORE, R3, ACCU), MI_ALU2(LOAD, SRCA, R2), MI_ALU2(LOAD, SRCB, R1), MI_ALU0(SUB), MI_ALU2(STORE, R1, ACCU), MI_ALU2(LOAD, SRCA, R3), MI_ALU2(LOAD, SRCB, R1), MI_ALU0(SUB), MI_ALU2(STORE, R1, ACCU), MI_ALU2(LOAD, SRCA, R1), MI_ALU2(LOAD, SRCB, R0), MI_ALU0(OR), MI_ALU2(STORE, R0, ACCU), }; iris_batch_emit(batch, maths, sizeof(maths)); } static void overflow_result_to_gpr0(struct iris_context *ice, struct iris_query *q) { struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; ice->vtbl.load_register_imm64(batch, CS_GPR(0), 0ull); if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) { load_overflow_data_to_cs_gprs(ice, q, q->index); calc_overflow_for_stream(ice); } else { for (int i = 0; i < MAX_VERTEX_STREAMS; i++) { load_overflow_data_to_cs_gprs(ice, q, i); calc_overflow_for_stream(ice); } } gpr0_to_bool(ice); } /* * GPR0 = GPR0 & ((1ull << n) -1); */ static void keep_gpr0_lower_n_bits(struct iris_context *ice, uint32_t n) { struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; ice->vtbl.load_register_imm64(batch, CS_GPR(1), (1ull << n) - 1); static const uint32_t math[] = { MI_MATH | (5 - 2), MI_ALU2(LOAD, SRCA, R0), MI_ALU2(LOAD, SRCB, R1), MI_ALU0(AND), MI_ALU2(STORE, R0, ACCU), }; iris_batch_emit(batch, math, sizeof(math)); } /* * GPR0 = GPR0 << 30; */ static void shl_gpr0_by_30_bits(struct iris_context *ice) { struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; /* First we mask 34 bits of GPR0 to prevent overflow */ keep_gpr0_lower_n_bits(ice, 34); static const uint32_t shl_math[] = { MI_ALU2(LOAD, SRCA, R0), MI_ALU2(LOAD, SRCB, R0), MI_ALU0(ADD), MI_ALU2(STORE, R0, ACCU), }; const uint32_t outer_count = 5; const uint32_t inner_count = 6; const uint32_t cmd_len = 1 + inner_count * ARRAY_SIZE(shl_math); const uint32_t batch_len = cmd_len * outer_count; uint32_t *map = iris_get_command_space(batch, batch_len * 4); uint32_t offset = 0; for (int o = 0; o < outer_count; o++) { map[offset++] = MI_MATH | (cmd_len - 2); for (int i = 0; i < inner_count; i++) { memcpy(&map[offset], shl_math, sizeof(shl_math)); offset += 4; } } } /* * GPR0 = GPR0 >> 2; * * Note that the upper 30 bits of GPR0 are lost! */ static void shr_gpr0_by_2_bits(struct iris_context *ice) { struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; shl_gpr0_by_30_bits(ice); ice->vtbl.load_register_reg32(batch, CS_GPR(0) + 4, CS_GPR(0)); ice->vtbl.load_register_imm32(batch, CS_GPR(0) + 4, 0); } /** * Calculate the result and store it to CS_GPR0. */ static void calculate_result_on_gpu(struct iris_context *ice, struct iris_query *q) { struct iris_batch *batch = &ice->batches[q->batch_idx]; struct iris_screen *screen = (void *) ice->ctx.screen; const struct gen_device_info *devinfo = &batch->screen->devinfo; struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res); uint32_t offset = q->query_state_ref.offset; if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE || q->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) { overflow_result_to_gpr0(ice, q); return; } if (q->type == PIPE_QUERY_TIMESTAMP) { ice->vtbl.load_register_mem64(batch, CS_GPR(0), bo, offset + offsetof(struct iris_query_snapshots, start)); /* TODO: This discards any fractional bits of the timebase scale. * We would need to do a bit of fixed point math on the CS ALU, or * launch an actual shader to calculate this with full precision. */ emit_mul_gpr0(batch, (1000000000ull / screen->devinfo.timestamp_frequency)); keep_gpr0_lower_n_bits(ice, 36); return; } ice->vtbl.load_register_mem64(batch, CS_GPR(1), bo, offset + offsetof(struct iris_query_snapshots, start)); ice->vtbl.load_register_mem64(batch, CS_GPR(2), bo, offset + offsetof(struct iris_query_snapshots, end)); static const uint32_t math[] = { MI_MATH | (5 - 2), MI_ALU2(LOAD, SRCA, R2), MI_ALU2(LOAD, SRCB, R1), MI_ALU0(SUB), MI_ALU2(STORE, R0, ACCU), }; iris_batch_emit(batch, math, sizeof(math)); /* WaDividePSInvocationCountBy4:HSW,BDW */ if (devinfo->gen == 8 && q->type == PIPE_QUERY_PIPELINE_STATISTICS_SINGLE && q->index == PIPE_STAT_QUERY_PS_INVOCATIONS) shr_gpr0_by_2_bits(ice); if (q->type == PIPE_QUERY_OCCLUSION_PREDICATE || q->type == PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE) gpr0_to_bool(ice); if (q->type == PIPE_QUERY_TIME_ELAPSED) { /* TODO: This discards fractional bits (see above). */ emit_mul_gpr0(batch, (1000000000ull / screen->devinfo.timestamp_frequency)); } } static struct pipe_query * iris_create_query(struct pipe_context *ctx, unsigned query_type, unsigned index) { struct iris_query *q = calloc(1, sizeof(struct iris_query)); q->type = query_type; q->index = index; if (q->type == PIPE_QUERY_PIPELINE_STATISTICS_SINGLE && q->index == PIPE_STAT_QUERY_CS_INVOCATIONS) q->batch_idx = IRIS_BATCH_COMPUTE; else q->batch_idx = IRIS_BATCH_RENDER; return (struct pipe_query *) q; } static void iris_destroy_query(struct pipe_context *ctx, struct pipe_query *p_query) { struct iris_query *query = (void *) p_query; struct iris_screen *screen = (void *) ctx->screen; iris_syncpt_reference(screen, &query->syncpt, NULL); free(query); } static boolean iris_begin_query(struct pipe_context *ctx, struct pipe_query *query) { struct iris_context *ice = (void *) ctx; struct iris_query *q = (void *) query; void *ptr = NULL; uint32_t size; if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE || q->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) size = sizeof(struct iris_query_so_overflow); else size = sizeof(struct iris_query_snapshots); u_upload_alloc(ice->query_buffer_uploader, 0, size, size, &q->query_state_ref.offset, &q->query_state_ref.res, &ptr); if (!iris_resource_bo(q->query_state_ref.res)) return false; q->map = ptr; if (!q->map) return false; q->result = 0ull; q->ready = false; WRITE_ONCE(q->map->snapshots_landed, false); if (q->type == PIPE_QUERY_PRIMITIVES_GENERATED && q->index == 0) { ice->state.prims_generated_query_active = true; ice->state.dirty |= IRIS_DIRTY_STREAMOUT | IRIS_DIRTY_CLIP; } if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE || q->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) write_overflow_values(ice, q, false); else write_value(ice, q, q->query_state_ref.offset + offsetof(struct iris_query_snapshots, start)); return true; } static bool iris_end_query(struct pipe_context *ctx, struct pipe_query *query) { struct iris_context *ice = (void *) ctx; struct iris_query *q = (void *) query; struct iris_batch *batch = &ice->batches[q->batch_idx]; if (q->type == PIPE_QUERY_TIMESTAMP) { iris_begin_query(ctx, query); iris_batch_reference_signal_syncpt(batch, &q->syncpt); mark_available(ice, q); return true; } if (q->type == PIPE_QUERY_PRIMITIVES_GENERATED && q->index == 0) { ice->state.prims_generated_query_active = false; ice->state.dirty |= IRIS_DIRTY_STREAMOUT | IRIS_DIRTY_CLIP; } if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE || q->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) write_overflow_values(ice, q, true); else write_value(ice, q, q->query_state_ref.offset + offsetof(struct iris_query_snapshots, end)); iris_batch_reference_signal_syncpt(batch, &q->syncpt); mark_available(ice, q); return true; } /** * See if the snapshots have landed for a query, and if so, compute the * result and mark it ready. Does not flush (unlike iris_get_query_result). */ static void iris_check_query_no_flush(struct iris_context *ice, struct iris_query *q) { struct iris_screen *screen = (void *) ice->ctx.screen; const struct gen_device_info *devinfo = &screen->devinfo; if (!q->ready && READ_ONCE(q->map->snapshots_landed)) { calculate_result_on_cpu(devinfo, q); } } static boolean iris_get_query_result(struct pipe_context *ctx, struct pipe_query *query, boolean wait, union pipe_query_result *result) { struct iris_context *ice = (void *) ctx; struct iris_query *q = (void *) query; struct iris_screen *screen = (void *) ctx->screen; const struct gen_device_info *devinfo = &screen->devinfo; struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res); if (!q->ready) { if (iris_batch_references(&ice->batches[q->batch_idx], bo)) iris_batch_flush(&ice->batches[q->batch_idx]); while (!READ_ONCE(q->map->snapshots_landed)) { if (wait) iris_wait_syncpt(ctx->screen, q->syncpt, INT64_MAX); else return false; } assert(READ_ONCE(q->map->snapshots_landed)); calculate_result_on_cpu(devinfo, q); } assert(q->ready); result->u64 = q->result; return true; } static void iris_get_query_result_resource(struct pipe_context *ctx, struct pipe_query *query, boolean wait, enum pipe_query_value_type result_type, int index, struct pipe_resource *p_res, unsigned offset) { struct iris_context *ice = (void *) ctx; struct iris_query *q = (void *) query; struct iris_batch *batch = &ice->batches[q->batch_idx]; const struct gen_device_info *devinfo = &batch->screen->devinfo; struct iris_resource *res = (void *) p_res; struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res); unsigned snapshots_landed_offset = offsetof(struct iris_query_snapshots, snapshots_landed); res->bind_history |= PIPE_BIND_QUERY_BUFFER; if (index == -1) { /* They're asking for the availability of the result. If we still * have commands queued up which produce the result, submit them * now so that progress happens. Either way, copy the snapshots * landed field to the destination resource. */ if (iris_batch_references(batch, bo)) iris_batch_flush(batch); ice->vtbl.copy_mem_mem(batch, iris_resource_bo(p_res), offset, bo, snapshots_landed_offset, result_type <= PIPE_QUERY_TYPE_U32 ? 4 : 8); return; } if (!q->ready && READ_ONCE(q->map->snapshots_landed)) { /* The final snapshots happen to have landed, so let's just compute * the result on the CPU now... */ calculate_result_on_cpu(devinfo, q); } if (q->ready) { /* We happen to have the result on the CPU, so just copy it. */ if (result_type <= PIPE_QUERY_TYPE_U32) { ice->vtbl.store_data_imm32(batch, iris_resource_bo(p_res), offset, q->result); } else { ice->vtbl.store_data_imm64(batch, iris_resource_bo(p_res), offset, q->result); } /* Make sure the result lands before they use bind the QBO elsewhere * and use the result. */ // XXX: Why? i965 doesn't do this. iris_emit_pipe_control_flush(batch, PIPE_CONTROL_CS_STALL); return; } /* Calculate the result to CS_GPR0 */ calculate_result_on_gpu(ice, q); bool predicated = !wait && !q->stalled; if (predicated) { ice->vtbl.load_register_imm64(batch, MI_PREDICATE_SRC1, 0ull); ice->vtbl.load_register_mem64(batch, MI_PREDICATE_SRC0, bo, snapshots_landed_offset); uint32_t predicate = MI_PREDICATE | MI_PREDICATE_LOADOP_LOADINV | MI_PREDICATE_COMBINEOP_SET | MI_PREDICATE_COMPAREOP_SRCS_EQUAL; iris_batch_emit(batch, &predicate, sizeof(uint32_t)); } if (result_type <= PIPE_QUERY_TYPE_U32) { ice->vtbl.store_register_mem32(batch, CS_GPR(0), iris_resource_bo(p_res), offset, predicated); } else { ice->vtbl.store_register_mem64(batch, CS_GPR(0), iris_resource_bo(p_res), offset, predicated); } } static void iris_set_active_query_state(struct pipe_context *ctx, boolean enable) { struct iris_context *ice = (void *) ctx; if (ice->state.statistics_counters_enabled == enable) return; // XXX: most packets aren't paying attention to this yet, because it'd // have to be done dynamically at draw time, which is a pain ice->state.statistics_counters_enabled = enable; ice->state.dirty |= IRIS_DIRTY_CLIP | IRIS_DIRTY_GS | IRIS_DIRTY_RASTER | IRIS_DIRTY_STREAMOUT | IRIS_DIRTY_TCS | IRIS_DIRTY_TES | IRIS_DIRTY_VS | IRIS_DIRTY_WM; } static void set_predicate_enable(struct iris_context *ice, bool value) { if (value) ice->state.predicate = IRIS_PREDICATE_STATE_RENDER; else ice->state.predicate = IRIS_PREDICATE_STATE_DONT_RENDER; } static void set_predicate_for_result(struct iris_context *ice, struct iris_query *q, bool inverted) { struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; struct iris_bo *bo = iris_resource_bo(q->query_state_ref.res); /* The CPU doesn't have the query result yet; use hardware predication */ ice->state.predicate = IRIS_PREDICATE_STATE_USE_BIT; /* Ensure the memory is coherent for MI_LOAD_REGISTER_* commands. */ iris_emit_pipe_control_flush(batch, PIPE_CONTROL_FLUSH_ENABLE); q->stalled = true; switch (q->type) { case PIPE_QUERY_SO_OVERFLOW_PREDICATE: case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE: overflow_result_to_gpr0(ice, q); ice->vtbl.load_register_reg64(batch, MI_PREDICATE_SRC0, CS_GPR(0)); ice->vtbl.load_register_imm64(batch, MI_PREDICATE_SRC1, 0ull); break; default: /* PIPE_QUERY_OCCLUSION_* */ ice->vtbl.load_register_mem64(batch, MI_PREDICATE_SRC0, bo, offsetof(struct iris_query_snapshots, start) + q->query_state_ref.offset); ice->vtbl.load_register_mem64(batch, MI_PREDICATE_SRC1, bo, offsetof(struct iris_query_snapshots, end) + q->query_state_ref.offset); break; } uint32_t mi_predicate = MI_PREDICATE | MI_PREDICATE_COMBINEOP_SET | MI_PREDICATE_COMPAREOP_SRCS_EQUAL | (inverted ? MI_PREDICATE_LOADOP_LOAD : MI_PREDICATE_LOADOP_LOADINV); iris_batch_emit(batch, &mi_predicate, sizeof(uint32_t)); /* We immediately set the predicate on the render batch, as all the * counters come from 3D operations. However, we may need to predicate * a compute dispatch, which executes in a different GEM context and has * a different MI_PREDICATE_RESULT register. So, we save the result to * memory and reload it in iris_launch_grid. */ unsigned offset = q->query_state_ref.offset + offsetof(struct iris_query_snapshots, predicate_result); ice->vtbl.store_register_mem64(batch, MI_PREDICATE_RESULT, bo, offset, false); ice->state.compute_predicate = bo; } static void iris_render_condition(struct pipe_context *ctx, struct pipe_query *query, boolean condition, enum pipe_render_cond_flag mode) { struct iris_context *ice = (void *) ctx; struct iris_query *q = (void *) query; /* The old condition isn't relevant; we'll update it if necessary */ ice->state.compute_predicate = NULL; ice->condition.query = q; ice->condition.condition = condition; if (!q) { ice->state.predicate = IRIS_PREDICATE_STATE_RENDER; return; } iris_check_query_no_flush(ice, q); if (q->result || q->ready) { set_predicate_enable(ice, (q->result != 0) ^ condition); } else { if (mode == PIPE_RENDER_COND_NO_WAIT || mode == PIPE_RENDER_COND_BY_REGION_NO_WAIT) { perf_debug(&ice->dbg, "Conditional rendering demoted from " "\"no wait\" to \"wait\"."); } set_predicate_for_result(ice, q, condition); } } void iris_resolve_conditional_render(struct iris_context *ice) { struct pipe_context *ctx = (void *) ice; struct iris_query *q = ice->condition.query; struct pipe_query *query = (void *) q; union pipe_query_result result; if (ice->state.predicate != IRIS_PREDICATE_STATE_USE_BIT) return; assert(q); iris_get_query_result(ctx, query, true, &result); set_predicate_enable(ice, (q->result != 0) ^ ice->condition.condition); } void iris_init_query_functions(struct pipe_context *ctx) { ctx->create_query = iris_create_query; ctx->destroy_query = iris_destroy_query; ctx->begin_query = iris_begin_query; ctx->end_query = iris_end_query; ctx->get_query_result = iris_get_query_result; ctx->get_query_result_resource = iris_get_query_result_resource; ctx->set_active_query_state = iris_set_active_query_state; ctx->render_condition = iris_render_condition; }