diff options
| -rw-r--r-- | src/compiler/Makefile.sources | 1 | ||||
| -rw-r--r-- | src/compiler/nir/meson.build | 1 | ||||
| -rw-r--r-- | src/compiler/nir/nir.h | 3 | ||||
| -rw-r--r-- | src/compiler/nir/nir_lower_alu.c | 165 | ||||
| -rw-r--r-- | src/mesa/state_tracker/st_glsl_to_nir.cpp | 1 |
5 files changed, 171 insertions, 0 deletions
diff --git a/src/compiler/Makefile.sources b/src/compiler/Makefile.sources index 3daa2c51334..d629c2b8ec8 100644 --- a/src/compiler/Makefile.sources +++ b/src/compiler/Makefile.sources @@ -214,6 +214,7 @@ NIR_FILES = \ nir/nir_loop_analyze.c \ nir/nir_loop_analyze.h \ nir/nir_lower_alpha_test.c \ + nir/nir_lower_alu.c \ nir/nir_lower_alu_to_scalar.c \ nir/nir_lower_atomics_to_ssbo.c \ nir/nir_lower_bitmap.c \ diff --git a/src/compiler/nir/meson.build b/src/compiler/nir/meson.build index 3fec363691d..598c68aff9f 100644 --- a/src/compiler/nir/meson.build +++ b/src/compiler/nir/meson.build @@ -104,6 +104,7 @@ files_libnir = files( 'nir_liveness.c', 'nir_loop_analyze.c', 'nir_loop_analyze.h', + 'nir_lower_alu.c', 'nir_lower_alu_to_scalar.c', 'nir_lower_alpha_test.c', 'nir_lower_atomics_to_ssbo.c', diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index b9426f8eb4e..7d01eb23bc4 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -1919,6 +1919,8 @@ typedef struct nir_shader_compiler_options { bool lower_find_lsb; bool lower_uadd_carry; bool lower_usub_borrow; + /** Lowers imul_high/umul_high to 16-bit multiplies and carry operations. */ + bool lower_mul_high; /** lowers fneg and ineg to fsub and isub. */ bool lower_negate; /** lowers fsub and isub to fadd+fneg and iadd+ineg. */ @@ -2628,6 +2630,7 @@ bool nir_move_vec_src_uses_to_dest(nir_shader *shader); bool nir_lower_vec_to_movs(nir_shader *shader); void nir_lower_alpha_test(nir_shader *shader, enum compare_func func, bool alpha_to_one); +bool nir_lower_alu(nir_shader *shader); bool nir_lower_alu_to_scalar(nir_shader *shader); bool nir_lower_load_const_to_scalar(nir_shader *shader); bool nir_lower_read_invocation_to_scalar(nir_shader *shader); diff --git a/src/compiler/nir/nir_lower_alu.c b/src/compiler/nir/nir_lower_alu.c new file mode 100644 index 00000000000..28ecaf6badc --- /dev/null +++ b/src/compiler/nir/nir_lower_alu.c @@ -0,0 +1,165 @@ +/* + * Copyright © 2010 Intel Corporation + * Copyright © 2018 Broadcom + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "nir.h" +#include "nir_builder.h" + +/** nir_lower_alu.c + * + * NIR's home for miscellaneous ALU operation lowering implementations. + * + * Most NIR ALU lowering occurs in nir_opt_algebraic.py, since it's generally + * easy to write them there. However, if terms appear multiple times in the + * lowered code, it can get very verbose and cause a lot of work for CSE, so + * it may end up being easier to write out in C code. + * + * The shader must be in SSA for this pass. + */ + +#define LOWER_MUL_HIGH (1 << 0) + +static bool +lower_alu_instr(nir_alu_instr *instr, nir_builder *b) +{ + nir_ssa_def *lowered = NULL; + + assert(instr->dest.dest.is_ssa); + + b->cursor = nir_before_instr(&instr->instr); + b->exact = instr->exact; + + switch (instr->op) { + case nir_op_imul_high: + case nir_op_umul_high: + if (b->shader->options->lower_mul_high) { + nir_ssa_def *c1 = nir_imm_int(b, 1); + nir_ssa_def *c16 = nir_imm_int(b, 16); + + nir_ssa_def *src0 = nir_ssa_for_alu_src(b, instr, 0); + nir_ssa_def *src1 = nir_ssa_for_alu_src(b, instr, 1); + nir_ssa_def *different_signs = NULL; + if (instr->op == nir_op_imul_high) { + nir_ssa_def *c0 = nir_imm_int(b, 0); + different_signs = nir_ixor(b, + nir_ilt(b, src0, c0), + nir_ilt(b, src1, c0)); + src0 = nir_iabs(b, src0); + src1 = nir_iabs(b, src1); + } + + /* ABCD + * * EFGH + * ====== + * (GH * CD) + (GH * AB) << 16 + (EF * CD) << 16 + (EF * AB) << 32 + * + * Start by splitting into the 4 multiplies. + */ + nir_ssa_def *src0l = nir_iand(b, src0, nir_imm_int(b, 0xffff)); + nir_ssa_def *src1l = nir_iand(b, src1, nir_imm_int(b, 0xffff)); + nir_ssa_def *src0h = nir_ushr(b, src0, c16); + nir_ssa_def *src1h = nir_ushr(b, src1, c16); + + nir_ssa_def *lo = nir_imul(b, src0l, src1l); + nir_ssa_def *m1 = nir_imul(b, src0l, src1h); + nir_ssa_def *m2 = nir_imul(b, src0h, src1l); + nir_ssa_def *hi = nir_imul(b, src0h, src1h); + + nir_ssa_def *tmp; + + tmp = nir_ishl(b, m1, c16); + hi = nir_iadd(b, hi, nir_iand(b, nir_uadd_carry(b, lo, tmp), c1)); + lo = nir_iadd(b, lo, tmp); + hi = nir_iadd(b, hi, nir_ushr(b, m1, c16)); + + tmp = nir_ishl(b, m2, c16); + hi = nir_iadd(b, hi, nir_iand(b, nir_uadd_carry(b, lo, tmp), c1)); + lo = nir_iadd(b, lo, tmp); + hi = nir_iadd(b, hi, nir_ushr(b, m2, c16)); + + if (instr->op == nir_op_imul_high) { + /* For channels where different_signs is set we have to perform a + * 64-bit negation. This is *not* the same as just negating the + * high 32-bits. Consider -3 * 2. The high 32-bits is 0, but the + * desired result is -1, not -0! Recall -x == ~x + 1. + */ + hi = nir_bcsel(b, different_signs, + nir_iadd(b, + nir_inot(b, hi), + nir_iand(b, + nir_uadd_carry(b, + nir_inot(b, lo), + c1), + nir_imm_int(b, 1))), + hi); + } + + lowered = hi; + } + break; + + default: + break; + } + + if (lowered) { + nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(lowered)); + nir_instr_remove(&instr->instr); + return true; + } else { + return false; + } +} + +bool +nir_lower_alu(nir_shader *shader) +{ + bool progress = false; + + if (!shader->options->lower_mul_high) + return false; + + nir_foreach_function(function, shader) { + if (function->impl) { + nir_builder builder; + nir_builder_init(&builder, function->impl); + + nir_foreach_block(block, function->impl) { + nir_foreach_instr_safe(instr, block) { + if (instr->type == nir_instr_type_alu) { + progress = lower_alu_instr(nir_instr_as_alu(instr), + &builder) || progress; + } + } + } + + if (progress) { + nir_metadata_preserve(function->impl, + nir_metadata_block_index | + nir_metadata_dominance); + } + } + } + + return progress; +} diff --git a/src/mesa/state_tracker/st_glsl_to_nir.cpp b/src/mesa/state_tracker/st_glsl_to_nir.cpp index de906e60c39..c8c2ef6cffd 100644 --- a/src/mesa/state_tracker/st_glsl_to_nir.cpp +++ b/src/mesa/state_tracker/st_glsl_to_nir.cpp @@ -325,6 +325,7 @@ st_nir_opts(nir_shader *nir) NIR_PASS_V(nir, nir_lower_alu_to_scalar); NIR_PASS_V(nir, nir_lower_phis_to_scalar); + NIR_PASS_V(nir, nir_lower_alu); NIR_PASS_V(nir, nir_lower_pack); NIR_PASS(progress, nir, nir_copy_prop); NIR_PASS(progress, nir, nir_opt_remove_phis); |