From b1d1b7222acfe45e447d1708d60b3ff98ca5a9a7 Mon Sep 17 00:00:00 2001 From: Jason Ekstrand Date: Fri, 6 Oct 2017 10:08:11 -0700 Subject: intel/compiler: Make brw_nir_lower_intrinsics compute-specific It's already only ever called from brw_compile_cs and only handles compute intrinsics. Let's just make it CS-specific. We can always make it handle other stages again later if we want. Reviewed-by: Jordan Justen Reviewed-by: Kenneth Graunke --- src/intel/Makefile.sources | 2 +- src/intel/compiler/brw_fs.cpp | 2 +- src/intel/compiler/brw_nir.h | 4 +- src/intel/compiler/brw_nir_intrinsics.c | 186 ----------------------- src/intel/compiler/brw_nir_lower_cs_intrinsics.c | 180 ++++++++++++++++++++++ src/intel/compiler/meson.build | 2 +- 6 files changed, 185 insertions(+), 191 deletions(-) delete mode 100644 src/intel/compiler/brw_nir_intrinsics.c create mode 100644 src/intel/compiler/brw_nir_lower_cs_intrinsics.c (limited to 'src') diff --git a/src/intel/Makefile.sources b/src/intel/Makefile.sources index 9672dcc252d..b8355337f83 100644 --- a/src/intel/Makefile.sources +++ b/src/intel/Makefile.sources @@ -75,7 +75,7 @@ COMPILER_FILES = \ compiler/brw_nir_analyze_boolean_resolves.c \ compiler/brw_nir_analyze_ubo_ranges.c \ compiler/brw_nir_attribute_workarounds.c \ - compiler/brw_nir_intrinsics.c \ + compiler/brw_nir_lower_cs_intrinsics.c \ compiler/brw_nir_opt_peephole_ffma.c \ compiler/brw_nir_tcs_workarounds.c \ compiler/brw_packed_float.c \ diff --git a/src/intel/compiler/brw_fs.cpp b/src/intel/compiler/brw_fs.cpp index 6f5f21ddcdf..371df710554 100644 --- a/src/intel/compiler/brw_fs.cpp +++ b/src/intel/compiler/brw_fs.cpp @@ -6766,7 +6766,7 @@ brw_compile_cs(const struct brw_compiler *compiler, void *log_data, MAX2(shader->num_uniforms, (unsigned)4 * (prog_data->thread_local_id_index + 1)); - brw_nir_lower_intrinsics(shader, &prog_data->base); + brw_nir_lower_cs_intrinsics(shader, prog_data); shader = brw_postprocess_nir(shader, compiler, true); prog_data->local_size[0] = shader->info.cs.local_size[0]; diff --git a/src/intel/compiler/brw_nir.h b/src/intel/compiler/brw_nir.h index f4b13b18c34..6f64a731cac 100644 --- a/src/intel/compiler/brw_nir.h +++ b/src/intel/compiler/brw_nir.h @@ -95,8 +95,8 @@ void brw_nir_analyze_boolean_resolves(nir_shader *nir); nir_shader *brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir); -bool brw_nir_lower_intrinsics(nir_shader *nir, - struct brw_stage_prog_data *prog_data); +bool brw_nir_lower_cs_intrinsics(nir_shader *nir, + struct brw_cs_prog_data *prog_data); void brw_nir_lower_vs_inputs(nir_shader *nir, bool use_legacy_snorm_formula, const uint8_t *vs_attrib_wa_flags); diff --git a/src/intel/compiler/brw_nir_intrinsics.c b/src/intel/compiler/brw_nir_intrinsics.c deleted file mode 100644 index abbbc6f93ec..00000000000 --- a/src/intel/compiler/brw_nir_intrinsics.c +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright (c) 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -#include "brw_nir.h" -#include "compiler/nir/nir_builder.h" - -struct lower_intrinsics_state { - nir_shader *nir; - union { - struct brw_stage_prog_data *prog_data; - struct brw_cs_prog_data *cs_prog_data; - }; - nir_function_impl *impl; - bool progress; - nir_builder builder; - bool cs_thread_id_used; -}; - -static nir_ssa_def * -read_thread_local_id(struct lower_intrinsics_state *state) -{ - nir_builder *b = &state->builder; - nir_shader *nir = state->nir; - const unsigned *sizes = nir->info.cs.local_size; - const unsigned group_size = sizes[0] * sizes[1] * sizes[2]; - - /* Some programs have local_size dimensions so small that the thread local - * ID will always be 0. - */ - if (group_size <= 8) - return nir_imm_int(b, 0); - - assert(state->cs_prog_data->thread_local_id_index >= 0); - state->cs_thread_id_used = true; - const int id_index = state->cs_prog_data->thread_local_id_index; - - nir_intrinsic_instr *load = - nir_intrinsic_instr_create(nir, nir_intrinsic_load_uniform); - load->num_components = 1; - load->src[0] = nir_src_for_ssa(nir_imm_int(b, 0)); - nir_ssa_dest_init(&load->instr, &load->dest, 1, 32, NULL); - nir_intrinsic_set_base(load, id_index * sizeof(uint32_t)); - nir_intrinsic_set_range(load, sizeof(uint32_t)); - nir_builder_instr_insert(b, &load->instr); - return &load->dest.ssa; -} - -static bool -lower_cs_intrinsics_convert_block(struct lower_intrinsics_state *state, - nir_block *block) -{ - bool progress = false; - nir_builder *b = &state->builder; - nir_shader *nir = state->nir; - - nir_foreach_instr_safe(instr, block) { - if (instr->type != nir_instr_type_intrinsic) - continue; - - nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr); - - b->cursor = nir_after_instr(&intrinsic->instr); - - nir_ssa_def *sysval; - switch (intrinsic->intrinsic) { - case nir_intrinsic_load_local_invocation_index: { - assert(nir->stage == MESA_SHADER_COMPUTE); - /* We construct the local invocation index from: - * - * gl_LocalInvocationIndex = - * cs_thread_local_id + subgroup_invocation; - */ - nir_ssa_def *thread_local_id = read_thread_local_id(state); - nir_ssa_def *channel = nir_load_subgroup_invocation(b); - sysval = nir_iadd(b, channel, thread_local_id); - break; - } - - case nir_intrinsic_load_local_invocation_id: { - assert(nir->stage == MESA_SHADER_COMPUTE); - /* We lower gl_LocalInvocationID from gl_LocalInvocationIndex based - * on this formula: - * - * gl_LocalInvocationID.x = - * gl_LocalInvocationIndex % gl_WorkGroupSize.x; - * gl_LocalInvocationID.y = - * (gl_LocalInvocationIndex / gl_WorkGroupSize.x) % - * gl_WorkGroupSize.y; - * gl_LocalInvocationID.z = - * (gl_LocalInvocationIndex / - * (gl_WorkGroupSize.x * gl_WorkGroupSize.y)) % - * gl_WorkGroupSize.z; - */ - unsigned *size = nir->info.cs.local_size; - - nir_ssa_def *local_index = nir_load_local_invocation_index(b); - - nir_const_value uvec3; - uvec3.u32[0] = 1; - uvec3.u32[1] = size[0]; - uvec3.u32[2] = size[0] * size[1]; - nir_ssa_def *div_val = nir_build_imm(b, 3, 32, uvec3); - uvec3.u32[0] = size[0]; - uvec3.u32[1] = size[1]; - uvec3.u32[2] = size[2]; - nir_ssa_def *mod_val = nir_build_imm(b, 3, 32, uvec3); - - sysval = nir_umod(b, nir_udiv(b, local_index, div_val), mod_val); - break; - } - - default: - continue; - } - - nir_ssa_def_rewrite_uses(&intrinsic->dest.ssa, nir_src_for_ssa(sysval)); - nir_instr_remove(&intrinsic->instr); - - state->progress = true; - } - - return progress; -} - -static void -lower_cs_intrinsics_convert_impl(struct lower_intrinsics_state *state) -{ - nir_builder_init(&state->builder, state->impl); - - nir_foreach_block(block, state->impl) { - lower_cs_intrinsics_convert_block(state, block); - } - - nir_metadata_preserve(state->impl, - nir_metadata_block_index | nir_metadata_dominance); -} - -bool -brw_nir_lower_intrinsics(nir_shader *nir, struct brw_stage_prog_data *prog_data) -{ - /* Currently we only lower intrinsics for compute shaders */ - if (nir->stage != MESA_SHADER_COMPUTE) - return false; - - bool progress = false; - struct lower_intrinsics_state state; - memset(&state, 0, sizeof(state)); - state.nir = nir; - state.prog_data = prog_data; - - do { - state.progress = false; - nir_foreach_function(function, nir) { - if (function->impl) { - state.impl = function->impl; - lower_cs_intrinsics_convert_impl(&state); - } - } - progress |= state.progress; - } while (state.progress); - - if (nir->stage == MESA_SHADER_COMPUTE && !state.cs_thread_id_used) - state.cs_prog_data->thread_local_id_index = -1; - - return progress; -} diff --git a/src/intel/compiler/brw_nir_lower_cs_intrinsics.c b/src/intel/compiler/brw_nir_lower_cs_intrinsics.c new file mode 100644 index 00000000000..602ef2e1749 --- /dev/null +++ b/src/intel/compiler/brw_nir_lower_cs_intrinsics.c @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "brw_nir.h" +#include "compiler/nir/nir_builder.h" + +struct lower_intrinsics_state { + nir_shader *nir; + struct brw_cs_prog_data *prog_data; + nir_function_impl *impl; + bool progress; + nir_builder builder; + bool cs_thread_id_used; +}; + +static nir_ssa_def * +read_thread_local_id(struct lower_intrinsics_state *state) +{ + nir_builder *b = &state->builder; + nir_shader *nir = state->nir; + const unsigned *sizes = nir->info.cs.local_size; + const unsigned group_size = sizes[0] * sizes[1] * sizes[2]; + + /* Some programs have local_size dimensions so small that the thread local + * ID will always be 0. + */ + if (group_size <= 8) + return nir_imm_int(b, 0); + + assert(state->prog_data->thread_local_id_index >= 0); + state->cs_thread_id_used = true; + const int id_index = state->prog_data->thread_local_id_index; + + nir_intrinsic_instr *load = + nir_intrinsic_instr_create(nir, nir_intrinsic_load_uniform); + load->num_components = 1; + load->src[0] = nir_src_for_ssa(nir_imm_int(b, 0)); + nir_ssa_dest_init(&load->instr, &load->dest, 1, 32, NULL); + nir_intrinsic_set_base(load, id_index * sizeof(uint32_t)); + nir_intrinsic_set_range(load, sizeof(uint32_t)); + nir_builder_instr_insert(b, &load->instr); + return &load->dest.ssa; +} + +static bool +lower_cs_intrinsics_convert_block(struct lower_intrinsics_state *state, + nir_block *block) +{ + bool progress = false; + nir_builder *b = &state->builder; + nir_shader *nir = state->nir; + + nir_foreach_instr_safe(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr); + + b->cursor = nir_after_instr(&intrinsic->instr); + + nir_ssa_def *sysval; + switch (intrinsic->intrinsic) { + case nir_intrinsic_load_local_invocation_index: { + /* We construct the local invocation index from: + * + * gl_LocalInvocationIndex = + * cs_thread_local_id + subgroup_invocation; + */ + nir_ssa_def *thread_local_id = read_thread_local_id(state); + nir_ssa_def *channel = nir_load_subgroup_invocation(b); + sysval = nir_iadd(b, channel, thread_local_id); + break; + } + + case nir_intrinsic_load_local_invocation_id: { + /* We lower gl_LocalInvocationID from gl_LocalInvocationIndex based + * on this formula: + * + * gl_LocalInvocationID.x = + * gl_LocalInvocationIndex % gl_WorkGroupSize.x; + * gl_LocalInvocationID.y = + * (gl_LocalInvocationIndex / gl_WorkGroupSize.x) % + * gl_WorkGroupSize.y; + * gl_LocalInvocationID.z = + * (gl_LocalInvocationIndex / + * (gl_WorkGroupSize.x * gl_WorkGroupSize.y)) % + * gl_WorkGroupSize.z; + */ + unsigned *size = nir->info.cs.local_size; + + nir_ssa_def *local_index = nir_load_local_invocation_index(b); + + nir_const_value uvec3; + uvec3.u32[0] = 1; + uvec3.u32[1] = size[0]; + uvec3.u32[2] = size[0] * size[1]; + nir_ssa_def *div_val = nir_build_imm(b, 3, 32, uvec3); + uvec3.u32[0] = size[0]; + uvec3.u32[1] = size[1]; + uvec3.u32[2] = size[2]; + nir_ssa_def *mod_val = nir_build_imm(b, 3, 32, uvec3); + + sysval = nir_umod(b, nir_udiv(b, local_index, div_val), mod_val); + break; + } + + default: + continue; + } + + nir_ssa_def_rewrite_uses(&intrinsic->dest.ssa, nir_src_for_ssa(sysval)); + nir_instr_remove(&intrinsic->instr); + + state->progress = true; + } + + return progress; +} + +static void +lower_cs_intrinsics_convert_impl(struct lower_intrinsics_state *state) +{ + nir_builder_init(&state->builder, state->impl); + + nir_foreach_block(block, state->impl) { + lower_cs_intrinsics_convert_block(state, block); + } + + nir_metadata_preserve(state->impl, + nir_metadata_block_index | nir_metadata_dominance); +} + +bool +brw_nir_lower_cs_intrinsics(nir_shader *nir, + struct brw_cs_prog_data *prog_data) +{ + assert(nir->stage == MESA_SHADER_COMPUTE); + + bool progress = false; + struct lower_intrinsics_state state; + memset(&state, 0, sizeof(state)); + state.nir = nir; + state.prog_data = prog_data; + + do { + state.progress = false; + nir_foreach_function(function, nir) { + if (function->impl) { + state.impl = function->impl; + lower_cs_intrinsics_convert_impl(&state); + } + } + progress |= state.progress; + } while (state.progress); + + if (!state.cs_thread_id_used) + state.prog_data->thread_local_id_index = -1; + + return progress; +} diff --git a/src/intel/compiler/meson.build b/src/intel/compiler/meson.build index 41c2f6ef1db..e29e1d39e2a 100644 --- a/src/intel/compiler/meson.build +++ b/src/intel/compiler/meson.build @@ -73,7 +73,7 @@ libintel_compiler_files = files( 'brw_nir_analyze_boolean_resolves.c', 'brw_nir_analyze_ubo_ranges.c', 'brw_nir_attribute_workarounds.c', - 'brw_nir_intrinsics.c', + 'brw_nir_lower_cs_intrinsics.c', 'brw_nir_opt_peephole_ffma.c', 'brw_nir_tcs_workarounds.c', 'brw_packed_float.c', -- cgit v1.2.3