diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/intel/Makefile.sources | 1 | ||||
| -rw-r--r-- | src/intel/compiler/brw_fs_nir.cpp | 381 | ||||
| -rw-r--r-- | src/intel/compiler/brw_nir.c | 2 | ||||
| -rw-r--r-- | src/intel/compiler/brw_nir.h | 2 | ||||
| -rw-r--r-- | src/intel/compiler/brw_nir_lower_mem_access_bit_sizes.c | 313 | ||||
| -rw-r--r-- | src/intel/compiler/brw_vec4_nir.cpp | 126 | ||||
| -rw-r--r-- | src/intel/compiler/meson.build | 1 |
7 files changed, 421 insertions, 405 deletions
diff --git a/src/intel/Makefile.sources b/src/intel/Makefile.sources index 4da887f7ed2..5e7d32293b7 100644 --- a/src/intel/Makefile.sources +++ b/src/intel/Makefile.sources @@ -85,6 +85,7 @@ COMPILER_FILES = \ compiler/brw_nir_attribute_workarounds.c \ compiler/brw_nir_lower_cs_intrinsics.c \ compiler/brw_nir_lower_image_load_store.c \ + compiler/brw_nir_lower_mem_access_bit_sizes.c \ compiler/brw_nir_opt_peephole_ffma.c \ compiler/brw_nir_tcs_workarounds.c \ compiler/brw_packed_float.c \ diff --git a/src/intel/compiler/brw_fs_nir.cpp b/src/intel/compiler/brw_fs_nir.cpp index 2b36171136e..84d0c6be6c3 100644 --- a/src/intel/compiler/brw_fs_nir.cpp +++ b/src/intel/compiler/brw_fs_nir.cpp @@ -26,6 +26,7 @@ #include "brw_fs_surface_builder.h" #include "brw_nir.h" #include "util/u_math.h" +#include "util/bitscan.h" using namespace brw; using namespace brw::surface_access; @@ -2250,107 +2251,6 @@ fs_visitor::get_indirect_offset(nir_intrinsic_instr *instr) return get_nir_src(*offset_src); } -static void -do_untyped_vector_read(const fs_builder &bld, - const fs_reg dest, - const fs_reg surf_index, - const fs_reg offset_reg, - unsigned num_components) -{ - if (type_sz(dest.type) <= 2) { - assert(dest.stride == 1); - boolean is_const_offset = offset_reg.file == BRW_IMMEDIATE_VALUE; - - if (is_const_offset) { - uint32_t start = offset_reg.ud & ~3; - uint32_t end = offset_reg.ud + num_components * type_sz(dest.type); - end = ALIGN(end, 4); - assert (end - start <= 16); - - /* At this point we have 16-bit component/s that have constant - * offset aligned to 4-bytes that can be read with untyped_reads. - * untyped_read message requires 32-bit aligned offsets. - */ - unsigned first_component = (offset_reg.ud & 3) / type_sz(dest.type); - unsigned num_components_32bit = (end - start) / 4; - - fs_reg read_result = - emit_untyped_read(bld, surf_index, brw_imm_ud(start), - 1 /* dims */, - num_components_32bit, - BRW_PREDICATE_NONE); - shuffle_from_32bit_read(bld, dest, read_result, first_component, - num_components); - } else { - fs_reg read_offset = bld.vgrf(BRW_REGISTER_TYPE_UD); - for (unsigned i = 0; i < num_components; i++) { - if (i == 0) { - bld.MOV(read_offset, offset_reg); - } else { - bld.ADD(read_offset, offset_reg, - brw_imm_ud(i * type_sz(dest.type))); - } - /* Non constant offsets are not guaranteed to be aligned 32-bits - * so they are read using one byte_scattered_read message - * for each component. - */ - fs_reg read_result = - emit_byte_scattered_read(bld, surf_index, read_offset, - 1 /* dims */, 1, - type_sz(dest.type) * 8 /* bit_size */, - BRW_PREDICATE_NONE); - bld.MOV(offset(dest, bld, i), - subscript (read_result, dest.type, 0)); - } - } - } else if (type_sz(dest.type) == 4) { - fs_reg read_result = emit_untyped_read(bld, surf_index, offset_reg, - 1 /* dims */, - num_components, - BRW_PREDICATE_NONE); - read_result.type = dest.type; - for (unsigned i = 0; i < num_components; i++) - bld.MOV(offset(dest, bld, i), offset(read_result, bld, i)); - } else if (type_sz(dest.type) == 8) { - /* Reading a dvec, so we need to: - * - * 1. Multiply num_components by 2, to account for the fact that we - * need to read 64-bit components. - * 2. Shuffle the result of the load to form valid 64-bit elements - * 3. Emit a second load (for components z/w) if needed. - */ - fs_reg read_offset = bld.vgrf(BRW_REGISTER_TYPE_UD); - bld.MOV(read_offset, offset_reg); - - int iters = num_components <= 2 ? 1 : 2; - - /* Load the dvec, the first iteration loads components x/y, the second - * iteration, if needed, loads components z/w - */ - for (int it = 0; it < iters; it++) { - /* Compute number of components to read in this iteration */ - int iter_components = MIN2(2, num_components); - num_components -= iter_components; - - /* Read. Since this message reads 32-bit components, we need to - * read twice as many components. - */ - fs_reg read_result = emit_untyped_read(bld, surf_index, read_offset, - 1 /* dims */, - iter_components * 2, - BRW_PREDICATE_NONE); - - /* Shuffle the 32-bit load result into valid 64-bit data */ - shuffle_from_32bit_read(bld, offset(dest, bld, it * 2), - read_result, 0, iter_components); - - bld.ADD(read_offset, read_offset, brw_imm_ud(16)); - } - } else { - unreachable("Unsupported type"); - } -} - void fs_visitor::nir_emit_vs_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) @@ -3572,93 +3472,64 @@ fs_visitor::nir_emit_cs_intrinsic(const fs_builder &bld, case nir_intrinsic_load_shared: { assert(devinfo->gen >= 7); + assert(stage == MESA_SHADER_COMPUTE); - fs_reg surf_index = brw_imm_ud(GEN7_BTI_SLM); + const unsigned bit_size = nir_dest_bit_size(instr->dest); + fs_reg offset_reg = retype(get_nir_src(instr->src[0]), + BRW_REGISTER_TYPE_UD); - /* Get the offset to read from */ - fs_reg offset_reg; - if (nir_src_is_const(instr->src[0])) { - offset_reg = brw_imm_ud(instr->const_index[0] + - nir_src_as_uint(instr->src[0])); - } else { - offset_reg = vgrf(glsl_type::uint_type); - bld.ADD(offset_reg, - retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD), - brw_imm_ud(instr->const_index[0])); - } + /* Make dest unsigned because that's what the temporary will be */ + dest.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); /* Read the vector */ - do_untyped_vector_read(bld, dest, surf_index, offset_reg, - instr->num_components); + if (nir_intrinsic_align(instr) >= 4) { + assert(nir_dest_bit_size(instr->dest) == 32); + fs_reg read_result = emit_untyped_read(bld, brw_imm_ud(GEN7_BTI_SLM), + offset_reg, 1 /* dims */, + instr->num_components, + BRW_PREDICATE_NONE); + for (unsigned i = 0; i < instr->num_components; i++) + bld.MOV(offset(dest, bld, i), offset(read_result, bld, i)); + } else { + assert(nir_dest_bit_size(instr->dest) <= 32); + assert(nir_dest_num_components(instr->dest) == 1); + fs_reg read_result = + emit_byte_scattered_read(bld, brw_imm_ud(GEN7_BTI_SLM), offset_reg, + 1 /* dims */, 1, bit_size, + BRW_PREDICATE_NONE); + bld.MOV(dest, read_result); + } break; } case nir_intrinsic_store_shared: { assert(devinfo->gen >= 7); + assert(stage == MESA_SHADER_COMPUTE); - /* Block index */ - fs_reg surf_index = brw_imm_ud(GEN7_BTI_SLM); - - /* Value */ + const unsigned bit_size = nir_src_bit_size(instr->src[0]); fs_reg val_reg = get_nir_src(instr->src[0]); + fs_reg offset_reg = retype(get_nir_src(instr->src[1]), + BRW_REGISTER_TYPE_UD); - /* Writemask */ - unsigned writemask = instr->const_index[1]; - - /* get_nir_src() retypes to integer. Be wary of 64-bit types though - * since the untyped writes below operate in units of 32-bits, which - * means that we need to write twice as many components each time. - * Also, we have to suffle 64-bit data to be in the appropriate layout - * expected by our 32-bit write messages. - */ - unsigned type_size = 4; - if (nir_src_bit_size(instr->src[0]) == 64) { - type_size = 8; - val_reg = shuffle_for_32bit_write(bld, val_reg, 0, - instr->num_components); - } - - unsigned type_slots = type_size / 4; - - /* Combine groups of consecutive enabled channels in one write - * message. We use ffs to find the first enabled channel and then ffs on - * the bit-inverse, down-shifted writemask to determine the length of - * the block of enabled bits. - */ - while (writemask) { - unsigned first_component = ffs(writemask) - 1; - unsigned length = ffs(~(writemask >> first_component)) - 1; - - /* We can't write more than 2 64-bit components at once. Limit the - * length of the write to what we can do and let the next iteration - * handle the rest - */ - if (type_size > 4) - length = MIN2(2, length); - - fs_reg offset_reg; - if (nir_src_is_const(instr->src[1])) { - offset_reg = brw_imm_ud(instr->const_index[0] + - nir_src_as_uint(instr->src[1]) + - type_size * first_component); - } else { - offset_reg = vgrf(glsl_type::uint_type); - bld.ADD(offset_reg, - retype(get_nir_src(instr->src[1]), BRW_REGISTER_TYPE_UD), - brw_imm_ud(instr->const_index[0] + type_size * first_component)); - } + val_reg.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); - emit_untyped_write(bld, surf_index, offset_reg, - offset(val_reg, bld, first_component * type_slots), - 1 /* dims */, length * type_slots, + assert(nir_intrinsic_write_mask(instr) == + (1 << instr->num_components) - 1); + if (nir_intrinsic_align(instr) >= 4) { + assert(nir_src_bit_size(instr->src[0]) == 32); + assert(nir_src_num_components(instr->src[0]) <= 4); + emit_untyped_write(bld, brw_imm_ud(GEN7_BTI_SLM), offset_reg, val_reg, + 1 /* dims */, instr->num_components, BRW_PREDICATE_NONE); - - /* Clear the bits in the writemask that we just wrote, then try - * again to see if more channels are left. - */ - writemask &= (15 << (first_component + length)); + } else { + assert(nir_src_bit_size(instr->src[0]) <= 32); + assert(nir_src_num_components(instr->src[0]) == 1); + fs_reg write_src = bld.vgrf(BRW_REGISTER_TYPE_UD); + bld.MOV(write_src, val_reg); + emit_byte_scattered_write(bld, brw_imm_ud(GEN7_BTI_SLM), offset_reg, + write_src, 1 /* dims */, bit_size, + BRW_PREDICATE_NONE); } - break; } @@ -4155,13 +4026,32 @@ fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr case nir_intrinsic_load_ssbo: { assert(devinfo->gen >= 7); + const unsigned bit_size = nir_dest_bit_size(instr->dest); fs_reg surf_index = get_nir_ssbo_intrinsic_index(bld, instr); - fs_reg offset_reg = get_nir_src_imm(instr->src[1]); + fs_reg offset_reg = retype(get_nir_src(instr->src[1]), + BRW_REGISTER_TYPE_UD); - /* Read the vector */ - do_untyped_vector_read(bld, dest, surf_index, offset_reg, - instr->num_components); + /* Make dest unsigned because that's what the temporary will be */ + dest.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); + /* Read the vector */ + if (nir_intrinsic_align(instr) >= 4) { + assert(nir_dest_bit_size(instr->dest) == 32); + fs_reg read_result = emit_untyped_read(bld, surf_index, offset_reg, + 1 /* dims */, + instr->num_components, + BRW_PREDICATE_NONE); + for (unsigned i = 0; i < instr->num_components; i++) + bld.MOV(offset(dest, bld, i), offset(read_result, bld, i)); + } else { + assert(nir_dest_bit_size(instr->dest) <= 32); + assert(nir_dest_num_components(instr->dest) == 1); + fs_reg read_result = + emit_byte_scattered_read(bld, surf_index, offset_reg, + 1 /* dims */, 1, bit_size, + BRW_PREDICATE_NONE); + bld.MOV(dest, read_result); + } break; } @@ -4171,125 +4061,30 @@ fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr if (stage == MESA_SHADER_FRAGMENT) brw_wm_prog_data(prog_data)->has_side_effects = true; - fs_reg surf_index = get_nir_ssbo_intrinsic_index(bld, instr); - - /* Value */ + const unsigned bit_size = nir_src_bit_size(instr->src[0]); fs_reg val_reg = get_nir_src(instr->src[0]); + fs_reg surf_index = get_nir_ssbo_intrinsic_index(bld, instr); + fs_reg offset_reg = retype(get_nir_src(instr->src[2]), + BRW_REGISTER_TYPE_UD); - /* Writemask */ - unsigned writemask = instr->const_index[0]; - - /* get_nir_src() retypes to integer. Be wary of 64-bit types though - * since the untyped writes below operate in units of 32-bits, which - * means that we need to write twice as many components each time. - * Also, we have to suffle 64-bit data to be in the appropriate layout - * expected by our 32-bit write messages. - */ - unsigned bit_size = nir_src_bit_size(instr->src[0]); - unsigned type_size = bit_size / 8; - - /* Combine groups of consecutive enabled channels in one write - * message. We use ffs to find the first enabled channel and then ffs on - * the bit-inverse, down-shifted writemask to determine the num_components - * of the block of enabled bits. - */ - while (writemask) { - unsigned first_component = ffs(writemask) - 1; - unsigned num_components = ffs(~(writemask >> first_component)) - 1; - fs_reg write_src = offset(val_reg, bld, first_component); - - if (type_size > 4) { - /* We can't write more than 2 64-bit components at once. Limit - * the num_components of the write to what we can do and let the next - * iteration handle the rest. - */ - num_components = MIN2(2, num_components); - write_src = shuffle_for_32bit_write(bld, write_src, 0, - num_components); - } else if (type_size < 4) { - /* For 16-bit types we pack two consecutive values into a 32-bit - * word and use an untyped write message. For single values or not - * 32-bit-aligned we need to use byte-scattered writes because - * untyped writes works with 32-bit components with 32-bit - * alignment. byte_scattered_write messages only support one - * 16-bit component at a time. As VK_KHR_relaxed_block_layout - * could be enabled we can not guarantee that not constant offsets - * to be 32-bit aligned for 16-bit types. For example an array, of - * 16-bit vec3 with array element stride of 6. - * - * In the case of 32-bit aligned constant offsets if there is - * a 3-components vector we submit one untyped-write message - * of 32-bit (first two components), and one byte-scattered - * write message (the last component). - */ - - if (!nir_src_is_const(instr->src[2]) || - ((nir_src_as_uint(instr->src[2]) + - type_size * first_component) % 4)) { - /* If we use a .yz writemask we also need to emit 2 - * byte-scattered write messages because of y-component not - * being aligned to 32-bit. - */ - num_components = 1; - } else if (num_components * type_size > 4 && - (num_components * type_size % 4)) { - /* If the pending components size is not a multiple of 4 bytes - * we left the not aligned components for following emits of - * length == 1 with byte_scattered_write. - */ - num_components -= (num_components * type_size % 4) / type_size; - } else if (num_components * type_size < 4) { - num_components = 1; - } - /* For num_components == 1 we are also shuffling the component - * because byte scattered writes of 16-bit need values to be dword - * aligned. Shuffling only one component would be the same as - * striding it. - */ - write_src = shuffle_for_32bit_write(bld, write_src, 0, - num_components); - } - - fs_reg offset_reg; - - if (nir_src_is_const(instr->src[2])) { - offset_reg = brw_imm_ud(nir_src_as_uint(instr->src[2]) + - type_size * first_component); - } else { - offset_reg = vgrf(glsl_type::uint_type); - bld.ADD(offset_reg, - retype(get_nir_src(instr->src[2]), BRW_REGISTER_TYPE_UD), - brw_imm_ud(type_size * first_component)); - } - - if (type_size < 4 && num_components == 1) { - /* Untyped Surface messages have a fixed 32-bit size, so we need - * to rely on byte scattered in order to write 16-bit elements. - * The byte_scattered_write message needs that every written 16-bit - * type to be aligned 32-bits (stride=2). - */ - emit_byte_scattered_write(bld, surf_index, offset_reg, - write_src, - 1 /* dims */, - bit_size, - BRW_PREDICATE_NONE); - } else { - assert(num_components * type_size <= 16); - assert((num_components * type_size) % 4 == 0); - assert(offset_reg.file != BRW_IMMEDIATE_VALUE || - offset_reg.ud % 4 == 0); - unsigned num_slots = (num_components * type_size) / 4; - - emit_untyped_write(bld, surf_index, offset_reg, - write_src, - 1 /* dims */, num_slots, - BRW_PREDICATE_NONE); - } + val_reg.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); - /* Clear the bits in the writemask that we just wrote, then try - * again to see if more channels are left. - */ - writemask &= (15 << (first_component + num_components)); + assert(nir_intrinsic_write_mask(instr) == + (1 << instr->num_components) - 1); + if (nir_intrinsic_align(instr) >= 4) { + assert(nir_src_bit_size(instr->src[0]) == 32); + assert(nir_src_num_components(instr->src[0]) <= 4); + emit_untyped_write(bld, surf_index, offset_reg, val_reg, + 1 /* dims */, instr->num_components, + BRW_PREDICATE_NONE); + } else { + assert(nir_src_bit_size(instr->src[0]) <= 32); + assert(nir_src_num_components(instr->src[0]) == 1); + fs_reg write_src = bld.vgrf(BRW_REGISTER_TYPE_UD); + bld.MOV(write_src, val_reg); + emit_byte_scattered_write(bld, surf_index, offset_reg, + write_src, 1 /* dims */, bit_size, + BRW_PREDICATE_NONE); } break; } diff --git a/src/intel/compiler/brw_nir.c b/src/intel/compiler/brw_nir.c index 10b03ef2fba..aa6788b9fe5 100644 --- a/src/intel/compiler/brw_nir.c +++ b/src/intel/compiler/brw_nir.c @@ -714,6 +714,8 @@ brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir) brw_nir_no_indirect_mask(compiler, nir->info.stage); OPT(nir_lower_indirect_derefs, indirect_mask); + OPT(brw_nir_lower_mem_access_bit_sizes); + /* Get rid of split copies */ nir = brw_nir_optimize(nir, compiler, is_scalar, false); diff --git a/src/intel/compiler/brw_nir.h b/src/intel/compiler/brw_nir.h index 2ff8c72b94f..bc81950d47e 100644 --- a/src/intel/compiler/brw_nir.h +++ b/src/intel/compiler/brw_nir.h @@ -119,6 +119,8 @@ bool brw_nir_lower_image_load_store(nir_shader *nir, void brw_nir_rewrite_image_intrinsic(nir_intrinsic_instr *intrin, nir_ssa_def *index); +bool brw_nir_lower_mem_access_bit_sizes(nir_shader *shader); + nir_shader *brw_postprocess_nir(nir_shader *nir, const struct brw_compiler *compiler, bool is_scalar); diff --git a/src/intel/compiler/brw_nir_lower_mem_access_bit_sizes.c b/src/intel/compiler/brw_nir_lower_mem_access_bit_sizes.c new file mode 100644 index 00000000000..a3320521f49 --- /dev/null +++ b/src/intel/compiler/brw_nir_lower_mem_access_bit_sizes.c @@ -0,0 +1,313 @@ +/* + * Copyright © 2018 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "brw_nir.h" +#include "compiler/nir/nir_builder.h" +#include "util/u_math.h" +#include "util/bitscan.h" + +static nir_ssa_def * +dup_mem_intrinsic(nir_builder *b, nir_intrinsic_instr *intrin, + nir_ssa_def *store_src, int offset, + unsigned num_components, unsigned bit_size, + unsigned align) +{ + const nir_intrinsic_info *info = &nir_intrinsic_infos[intrin->intrinsic]; + + nir_intrinsic_instr *dup = + nir_intrinsic_instr_create(b->shader, intrin->intrinsic); + + nir_src *intrin_offset_src = nir_get_io_offset_src(intrin); + for (unsigned i = 0; i < info->num_srcs; i++) { + assert(intrin->src[i].is_ssa); + if (i == 0 && store_src) { + assert(!info->has_dest); + assert(&intrin->src[i] != intrin_offset_src); + dup->src[i] = nir_src_for_ssa(store_src); + } else if (&intrin->src[i] == intrin_offset_src) { + dup->src[i] = nir_src_for_ssa(nir_iadd_imm(b, intrin->src[i].ssa, + offset)); + } else { + dup->src[i] = nir_src_for_ssa(intrin->src[i].ssa); + } + } + + dup->num_components = num_components; + + for (unsigned i = 0; i < info->num_indices; i++) + dup->const_index[i] = intrin->const_index[i]; + + nir_intrinsic_set_align(dup, align, 0); + + if (info->has_dest) { + assert(intrin->dest.is_ssa); + nir_ssa_dest_init(&dup->instr, &dup->dest, + num_components, bit_size, + intrin->dest.ssa.name); + } else { + nir_intrinsic_set_write_mask(dup, (1 << num_components) - 1); + } + + nir_builder_instr_insert(b, &dup->instr); + + return info->has_dest ? &dup->dest.ssa : NULL; +} + +static bool +lower_mem_load_bit_size(nir_builder *b, nir_intrinsic_instr *intrin) +{ + assert(intrin->dest.is_ssa); + if (intrin->dest.ssa.bit_size == 32) + return false; + + const unsigned bit_size = intrin->dest.ssa.bit_size; + const unsigned num_components = intrin->dest.ssa.num_components; + const unsigned bytes_read = num_components * (bit_size / 8); + const unsigned align = nir_intrinsic_align(intrin); + + nir_ssa_def *result[4] = { NULL, }; + + nir_src *offset_src = nir_get_io_offset_src(intrin); + if (bit_size < 32 && nir_src_is_const(*offset_src)) { + /* The offset is constant so we can use a 32-bit load and just shift it + * around as needed. + */ + const int load_offset = nir_src_as_uint(*offset_src) % 4; + assert(load_offset % (bit_size / 8) == 0); + const unsigned load_comps32 = DIV_ROUND_UP(bytes_read + load_offset, 4); + /* A 16-bit vec4 is a 32-bit vec2. We add an extra component in case + * we offset into a component with load_offset. + */ + assert(load_comps32 <= 3); + + nir_ssa_def *load = dup_mem_intrinsic(b, intrin, NULL, -load_offset, + load_comps32, 32, 4); + nir_ssa_def *unpacked[3]; + for (unsigned i = 0; i < load_comps32; i++) + unpacked[i] = nir_unpack_bits(b, nir_channel(b, load, i), bit_size); + + assert(load_offset % (bit_size / 8) == 0); + const unsigned divisor = 32 / bit_size; + + for (unsigned i = 0; i < num_components; i++) { + unsigned load_i = i + load_offset / (bit_size / 8); + result[i] = nir_channel(b, unpacked[load_i / divisor], + load_i % divisor); + } + } else { + /* Otherwise, we have to break it into smaller loads */ + unsigned res_idx = 0; + int load_offset = 0; + while (load_offset < bytes_read) { + const unsigned bytes_left = bytes_read - load_offset; + unsigned load_bit_size, load_comps; + if (align < 4) { + load_comps = 1; + /* Choose a byte, word, or dword */ + load_bit_size = util_next_power_of_two(MIN2(bytes_left, 4)) * 8; + } else { + assert(load_offset % 4 == 0); + load_bit_size = 32; + load_comps = DIV_ROUND_UP(MIN2(bytes_left, 16), 4); + } + + nir_ssa_def *load = dup_mem_intrinsic(b, intrin, NULL, load_offset, + load_comps, load_bit_size, + align); + + nir_ssa_def *unpacked = nir_bitcast_vector(b, load, bit_size); + for (unsigned i = 0; i < unpacked->num_components; i++) { + if (res_idx < num_components) + result[res_idx++] = nir_channel(b, unpacked, i); + } + + load_offset += load_comps * (load_bit_size / 8); + } + } + + nir_ssa_def *vec_result = nir_vec(b, result, num_components); + nir_ssa_def_rewrite_uses(&intrin->dest.ssa, + nir_src_for_ssa(vec_result)); + nir_instr_remove(&intrin->instr); + + return true; +} + +static bool +lower_mem_store_bit_size(nir_builder *b, nir_intrinsic_instr *intrin) +{ + assert(intrin->src[0].is_ssa); + nir_ssa_def *value = intrin->src[0].ssa; + + assert(intrin->num_components == value->num_components); + const unsigned bit_size = value->bit_size; + const unsigned num_components = intrin->num_components; + const unsigned bytes_written = num_components * (bit_size / 8); + const unsigned align_mul = nir_intrinsic_align_mul(intrin); + const unsigned align_offset = nir_intrinsic_align_offset(intrin); + const unsigned align = nir_intrinsic_align(intrin); + + nir_component_mask_t writemask = nir_intrinsic_write_mask(intrin); + assert(writemask < (1 << num_components)); + + if ((value->bit_size <= 32 && num_components == 1) || + (value->bit_size == 32 && writemask == (1 << num_components) - 1)) + return false; + + nir_src *offset_src = nir_get_io_offset_src(intrin); + const bool offset_is_const = nir_src_is_const(*offset_src); + const unsigned const_offset = + offset_is_const ? nir_src_as_uint(*offset_src) : 0; + + assert(num_components * (bit_size / 8) <= 32); + uint32_t byte_mask = 0; + for (unsigned i = 0; i < num_components; i++) { + if (writemask & (1 << i)) + byte_mask |= ((1 << (bit_size / 8)) - 1) << i * (bit_size / 8); + } + + while (byte_mask) { + const int start = ffs(byte_mask) - 1; + assert(start % (bit_size / 8) == 0); + + int end; + for (end = start + 1; end < bytes_written; end++) { + if (!(byte_mask & (1 << end))) + break; + } + /* The size of the current contiguous chunk in bytes */ + const unsigned chunk_bytes = end - start; + + const bool is_dword_aligned = + (align_mul >= 4 && (align_offset + start) % 4 == 0) || + (offset_is_const && (start + const_offset) % 4 == 0); + + unsigned store_comps, store_bit_size, store_align; + if (chunk_bytes >= 4 && is_dword_aligned) { + store_align = MAX2(align, 4); + store_bit_size = 32; + store_comps = MIN2(chunk_bytes, 16) / 4; + } else { + store_align = align; + store_comps = 1; + store_bit_size = MIN2(chunk_bytes, 4) * 8; + /* The bit size must be a power of two */ + if (store_bit_size == 24) + store_bit_size = 16; + } + + const unsigned store_bytes = store_comps * (store_bit_size / 8); + assert(store_bytes % (bit_size / 8) == 0); + const unsigned store_first_src_comp = start / (bit_size / 8); + const unsigned store_src_comps = store_bytes / (bit_size / 8); + assert(store_first_src_comp + store_src_comps <= num_components); + + unsigned src_swiz[4]; + for (unsigned i = 0; i < store_src_comps; i++) + src_swiz[i] = store_first_src_comp + i; + nir_ssa_def *store_value = + nir_swizzle(b, value, src_swiz, store_src_comps, false); + nir_ssa_def *packed = nir_bitcast_vector(b, store_value, store_bit_size); + + dup_mem_intrinsic(b, intrin, packed, start, + store_comps, store_bit_size, store_align); + + byte_mask &= ~(((1u << store_bytes) - 1) << start); + } + + nir_instr_remove(&intrin->instr); + + return true; +} + +static bool +lower_mem_access_bit_sizes_impl(nir_function_impl *impl) +{ + bool progress = false; + + nir_builder b; + nir_builder_init(&b, impl); + + nir_foreach_block(block, impl) { + nir_foreach_instr_safe(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + b.cursor = nir_after_instr(instr); + + nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); + switch (intrin->intrinsic) { + case nir_intrinsic_load_ssbo: + case nir_intrinsic_load_shared: + if (lower_mem_load_bit_size(&b, intrin)) + progress = true; + break; + + case nir_intrinsic_store_ssbo: + case nir_intrinsic_store_shared: + if (lower_mem_store_bit_size(&b, intrin)) + progress = true; + break; + + default: + break; + } + } + } + + if (progress) { + nir_metadata_preserve(impl, nir_metadata_block_index | + nir_metadata_dominance); + } + + return progress; +} + +/** + * This pass loads arbitrary SSBO and shared memory load/store operations to + * intrinsics which are natively handleable by GEN hardware. In particular, + * we have two general types of memory load/store messages: + * + * - Untyped surface read/write: These can load/store between one and four + * dword components to/from a dword-aligned offset. + * + * - Byte scattered read/write: These can load/store a single byte, word, or + * dword scalar to/from an unaligned byte offset. + * + * Neither type of message can do a write-masked store. This pass converts + * all nir load/store intrinsics into a series of either 8 or 32-bit + * load/store intrinsics with a number of components that we can directly + * handle in hardware and with a trivial write-mask. + */ +bool +brw_nir_lower_mem_access_bit_sizes(nir_shader *shader) +{ + bool progress = false; + + nir_foreach_function(func, shader) { + if (func->impl && lower_mem_access_bit_sizes_impl(func->impl)) + progress = true; + } + + return progress; +} diff --git a/src/intel/compiler/brw_vec4_nir.cpp b/src/intel/compiler/brw_vec4_nir.cpp index 564be7e5eee..26ca2ddd8dc 100644 --- a/src/intel/compiler/brw_vec4_nir.cpp +++ b/src/intel/compiler/brw_vec4_nir.cpp @@ -500,6 +500,11 @@ vec4_visitor::nir_emit_intrinsic(nir_intrinsic_instr *instr) case nir_intrinsic_store_ssbo: { assert(devinfo->gen >= 7); + /* brw_nir_lower_mem_access_bit_sizes takes care of this */ + assert(nir_src_bit_size(instr->src[0]) == 32); + assert(nir_intrinsic_write_mask(instr) == + (1 << instr->num_components) - 1); + src_reg surf_index = get_nir_ssbo_intrinsic_index(instr); src_reg offset_reg = retype(get_nir_src_imm(instr->src[2]), BRW_REGISTER_TYPE_UD); @@ -507,9 +512,6 @@ vec4_visitor::nir_emit_intrinsic(nir_intrinsic_instr *instr) /* Value */ src_reg val_reg = get_nir_src(instr->src[0], BRW_REGISTER_TYPE_F, 4); - /* Writemask */ - unsigned write_mask = instr->const_index[0]; - /* IvyBridge does not have a native SIMD4x2 untyped write message so untyped * writes will use SIMD8 mode. In order to hide this and keep symmetry across * typed and untyped messages and across hardware platforms, the @@ -551,92 +553,18 @@ vec4_visitor::nir_emit_intrinsic(nir_intrinsic_instr *instr) const vec4_builder bld = vec4_builder(this).at_end() .annotate(current_annotation, base_ir); - unsigned type_slots = nir_src_bit_size(instr->src[0]) / 32; - if (type_slots == 2) { - dst_reg tmp = dst_reg(this, glsl_type::dvec4_type); - shuffle_64bit_data(tmp, retype(val_reg, tmp.type), true); - val_reg = src_reg(retype(tmp, BRW_REGISTER_TYPE_F)); - } - - uint8_t swizzle[4] = { 0, 0, 0, 0}; - int num_channels = 0; - unsigned skipped_channels = 0; - int num_components = instr->num_components; - for (int i = 0; i < num_components; i++) { - /* Read components Z/W of a dvec from the appropriate place. We will - * also have to adjust the swizzle (we do that with the '% 4' below) - */ - if (i == 2 && type_slots == 2) - val_reg = byte_offset(val_reg, REG_SIZE); - - /* Check if this channel needs to be written. If so, record the - * channel we need to take the data from in the swizzle array - */ - int component_mask = 1 << i; - int write_test = write_mask & component_mask; - if (write_test) { - /* If we are writing doubles we have to write 2 channels worth of - * of data (64 bits) for each double component. - */ - swizzle[num_channels++] = (i * type_slots) % 4; - if (type_slots == 2) - swizzle[num_channels++] = (i * type_slots + 1) % 4; - } - - /* If we don't have to write this channel it means we have a gap in the - * vector, so write the channels we accumulated until now, if any. Do - * the same if this was the last component in the vector, if we have - * enough channels for a full vec4 write or if we have processed - * components XY of a dvec (since components ZW are not in the same - * SIMD register) - */ - if (!write_test || i == num_components - 1 || num_channels == 4 || - (i == 1 && type_slots == 2)) { - if (num_channels > 0) { - /* We have channels to write, so update the offset we need to - * write at to skip the channels we skipped, if any. - */ - if (skipped_channels > 0) { - if (offset_reg.file == IMM) { - offset_reg.ud += 4 * skipped_channels; - } else { - emit(ADD(dst_reg(offset_reg), offset_reg, - brw_imm_ud(4 * skipped_channels))); - } - } - - /* Swizzle the data register so we take the data from the channels - * we need to write and send the write message. This will write - * num_channels consecutive dwords starting at offset. - */ - val_reg.swizzle = - BRW_SWIZZLE4(swizzle[0], swizzle[1], swizzle[2], swizzle[3]); - emit_untyped_write(bld, surf_index, offset_reg, val_reg, - 1 /* dims */, num_channels /* size */, - BRW_PREDICATE_NONE); - - /* If we have to do a second write we will have to update the - * offset so that we jump over the channels we have just written - * now. - */ - skipped_channels = num_channels; - - /* Restart the count for the next write message */ - num_channels = 0; - } - - /* If we didn't write the channel, increase skipped count */ - if (!write_test) - skipped_channels += type_slots; - } - } - + emit_untyped_write(bld, surf_index, offset_reg, val_reg, + 1 /* dims */, instr->num_components /* size */, + BRW_PREDICATE_NONE); break; } case nir_intrinsic_load_ssbo: { assert(devinfo->gen >= 7); + /* brw_nir_lower_mem_access_bit_sizes takes care of this */ + assert(nir_dest_bit_size(instr->dest) == 32); + src_reg surf_index = get_nir_ssbo_intrinsic_index(instr); src_reg offset_reg = retype(get_nir_src_imm(instr->src[1]), BRW_REGISTER_TYPE_UD); @@ -645,36 +573,10 @@ vec4_visitor::nir_emit_intrinsic(nir_intrinsic_instr *instr) const vec4_builder bld = vec4_builder(this).at_end() .annotate(current_annotation, base_ir); - src_reg read_result; + src_reg read_result = emit_untyped_read(bld, surf_index, offset_reg, + 1 /* dims */, 4 /* size*/, + BRW_PREDICATE_NONE); dst_reg dest = get_nir_dest(instr->dest); - if (type_sz(dest.type) < 8) { - read_result = emit_untyped_read(bld, surf_index, offset_reg, - 1 /* dims */, 4 /* size*/, - BRW_PREDICATE_NONE); - } else { - src_reg shuffled = src_reg(this, glsl_type::dvec4_type); - - src_reg temp; - temp = emit_untyped_read(bld, surf_index, offset_reg, - 1 /* dims */, 4 /* size*/, - BRW_PREDICATE_NONE); - emit(MOV(dst_reg(retype(shuffled, temp.type)), temp)); - - if (offset_reg.file == IMM) - offset_reg.ud += 16; - else - emit(ADD(dst_reg(offset_reg), offset_reg, brw_imm_ud(16))); - - temp = emit_untyped_read(bld, surf_index, offset_reg, - 1 /* dims */, 4 /* size*/, - BRW_PREDICATE_NONE); - emit(MOV(dst_reg(retype(byte_offset(shuffled, REG_SIZE), temp.type)), - temp)); - - read_result = src_reg(this, glsl_type::dvec4_type); - shuffle_64bit_data(dst_reg(read_result), shuffled, false); - } - read_result.type = dest.type; read_result.swizzle = brw_swizzle_for_size(instr->num_components); emit(MOV(dest, read_result)); diff --git a/src/intel/compiler/meson.build b/src/intel/compiler/meson.build index 3cdeb6214a8..953e8dcc971 100644 --- a/src/intel/compiler/meson.build +++ b/src/intel/compiler/meson.build @@ -78,6 +78,7 @@ libintel_compiler_files = files( 'brw_nir_attribute_workarounds.c', 'brw_nir_lower_cs_intrinsics.c', 'brw_nir_lower_image_load_store.c', + 'brw_nir_lower_mem_access_bit_sizes.c', 'brw_nir_opt_peephole_ffma.c', 'brw_nir_tcs_workarounds.c', 'brw_packed_float.c', |