diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/compiler/nir/nir.h | 4 | ||||
-rw-r--r-- | src/compiler/nir/nir_lower_int64.c | 29 | ||||
-rw-r--r-- | src/compiler/nir/nir_opcodes.py | 6 | ||||
-rw-r--r-- | src/compiler/nir/nir_opt_algebraic.py | 2 | ||||
-rw-r--r-- | src/intel/compiler/brw_compiler.c | 7 | ||||
-rw-r--r-- | src/intel/compiler/brw_fs_nir.cpp | 5 |
6 files changed, 47 insertions, 6 deletions
diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 0c96bdfbc56..0b10fb2e2b4 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -2118,6 +2118,7 @@ typedef enum { nir_lower_logic64 = (1 << 9), nir_lower_minmax64 = (1 << 10), nir_lower_shift64 = (1 << 11), + nir_lower_imul_2x32_64 = (1 << 12), } nir_lower_int64_options; typedef enum { @@ -2259,6 +2260,9 @@ typedef struct nir_shader_compiler_options { */ bool use_interpolated_input_intrinsics; + /* Lowers when 32x32->64 bit multiplication is not supported */ + bool lower_mul_2x32_64; + unsigned max_unroll_iterations; nir_lower_int64_options lower_int64_options; diff --git a/src/compiler/nir/nir_lower_int64.c b/src/compiler/nir/nir_lower_int64.c index 1c4b4b33797..6aae1816bd2 100644 --- a/src/compiler/nir/nir_lower_int64.c +++ b/src/compiler/nir/nir_lower_int64.c @@ -384,6 +384,16 @@ lower_imin64(nir_builder *b, nir_ssa_def *x, nir_ssa_def *y) } static nir_ssa_def * +lower_mul_2x32_64(nir_builder *b, nir_ssa_def *x, nir_ssa_def *y, + bool sign_extend) +{ + nir_ssa_def *res_hi = sign_extend ? nir_imul_high(b, x, y) + : nir_umul_high(b, x, y); + + return nir_pack_64_2x32_split(b, nir_imul(b, x, y), res_hi); +} + +static nir_ssa_def * lower_imul64(nir_builder *b, nir_ssa_def *x, nir_ssa_def *y) { nir_ssa_def *x_lo = nir_unpack_64_2x32_split_x(b, x); @@ -391,12 +401,13 @@ lower_imul64(nir_builder *b, nir_ssa_def *x, nir_ssa_def *y) nir_ssa_def *y_lo = nir_unpack_64_2x32_split_x(b, y); nir_ssa_def *y_hi = nir_unpack_64_2x32_split_y(b, y); - nir_ssa_def *res_lo = nir_imul(b, x_lo, y_lo); - nir_ssa_def *res_hi = nir_iadd(b, nir_umul_high(b, x_lo, y_lo), + nir_ssa_def *mul_lo = nir_umul_2x32_64(b, x_lo, y_lo); + nir_ssa_def *res_hi = nir_iadd(b, nir_unpack_64_2x32_split_y(b, mul_lo), nir_iadd(b, nir_imul(b, x_lo, y_hi), nir_imul(b, x_hi, y_lo))); - return nir_pack_64_2x32_split(b, res_lo, res_hi); + return nir_pack_64_2x32_split(b, nir_unpack_64_2x32_split_x(b, mul_lo), + res_hi); } static nir_ssa_def * @@ -441,9 +452,8 @@ lower_mul_high64(nir_builder *b, nir_ssa_def *x, nir_ssa_def *y, * so we're guaranteed that we can add in two more 32-bit values * without overflowing tmp. */ - nir_ssa_def *tmp = - nir_pack_64_2x32_split(b, nir_imul(b, x32[i], y32[j]), - nir_umul_high(b, x32[i], y32[j])); + nir_ssa_def *tmp = nir_umul_2x32_64(b, x32[i], y32[i]); + if (res[i + j]) tmp = nir_iadd(b, tmp, nir_u2u64(b, res[i + j])); if (carry) @@ -626,6 +636,9 @@ opcode_to_options_mask(nir_op opcode) switch (opcode) { case nir_op_imul: return nir_lower_imul64; + case nir_op_imul_2x32_64: + case nir_op_umul_2x32_64: + return nir_lower_imul_2x32_64; case nir_op_imul_high: case nir_op_umul_high: return nir_lower_imul_high64; @@ -688,6 +701,10 @@ lower_int64_alu_instr(nir_builder *b, nir_alu_instr *alu) switch (alu->op) { case nir_op_imul: return lower_imul64(b, src[0], src[1]); + case nir_op_imul_2x32_64: + return lower_mul_2x32_64(b, src[0], src[1], true); + case nir_op_umul_2x32_64: + return lower_mul_2x32_64(b, src[0], src[1], false); case nir_op_imul_high: return lower_mul_high64(b, src[0], src[1], true); case nir_op_umul_high: diff --git a/src/compiler/nir/nir_opcodes.py b/src/compiler/nir/nir_opcodes.py index 499deb947e8..42f8662352e 100644 --- a/src/compiler/nir/nir_opcodes.py +++ b/src/compiler/nir/nir_opcodes.py @@ -475,6 +475,12 @@ binop("fmul", tfloat, commutative + associative, "src0 * src1") # low 32-bits of signed/unsigned integer multiply binop("imul", tint, commutative + associative, "src0 * src1") +# Generate 64 bit result from 2 32 bits quantity +binop_convert("imul_2x32_64", tint64, tint32, commutative, + "(int64_t)src0 * (int64_t)src1") +binop_convert("umul_2x32_64", tuint64, tuint32, commutative, + "(uint64_t)src0 * (uint64_t)src1") + # high 32-bits of signed integer multiply binop("imul_high", tint, commutative, """ if (bit_size == 64) { diff --git a/src/compiler/nir/nir_opt_algebraic.py b/src/compiler/nir/nir_opt_algebraic.py index 617ca0ea933..53cfa94ae93 100644 --- a/src/compiler/nir/nir_opt_algebraic.py +++ b/src/compiler/nir/nir_opt_algebraic.py @@ -70,6 +70,8 @@ optimizations = [ (('imul', a, '#b@32(is_pos_power_of_two)'), ('ishl', a, ('find_lsb', b))), (('imul', a, '#b@32(is_neg_power_of_two)'), ('ineg', ('ishl', a, ('find_lsb', ('iabs', b))))), + (('imul_2x32_64', a, b), ('pack_64_2x32_split', ('imul', a, b), ('imul_high', a, b)), 'options->lower_mul_2x32_64'), + (('umul_2x32_64', a, b), ('pack_64_2x32_split', ('imul', a, b), ('umul_high', a, b)), 'options->lower_mul_2x32_64'), (('udiv', a, 1), a), (('idiv', a, 1), a), (('umod', a, 1), 0), diff --git a/src/intel/compiler/brw_compiler.c b/src/intel/compiler/brw_compiler.c index b3df0d9fa23..28793b1f0e6 100644 --- a/src/intel/compiler/brw_compiler.c +++ b/src/intel/compiler/brw_compiler.c @@ -171,6 +171,13 @@ brw_compiler_create(void *mem_ctx, const struct gen_device_info *devinfo) fp64_options |= nir_lower_fp64_full_software; } + /* The Bspec's section tittled "Instruction_multiply[DevBDW+]" claims that + * destination type can be Quadword and source type Doubleword for Gen8 and + * Gen9. So, lower 64 bit multiply instruction on rest of the platforms. + */ + if (devinfo->gen < 8 || devinfo->gen > 9) + int64_options |= nir_lower_imul_2x32_64; + /* We want the GLSL compiler to emit code that uses condition codes */ for (int i = 0; i < MESA_SHADER_STAGES; i++) { compiler->glsl_compiler_options[i].MaxUnrollIterations = 0; diff --git a/src/intel/compiler/brw_fs_nir.cpp b/src/intel/compiler/brw_fs_nir.cpp index a4fd29ff9ec..a7abaf742e2 100644 --- a/src/intel/compiler/brw_fs_nir.cpp +++ b/src/intel/compiler/brw_fs_nir.cpp @@ -1055,6 +1055,11 @@ fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr) inst->saturate = instr->dest.saturate; break; + case nir_op_imul_2x32_64: + case nir_op_umul_2x32_64: + bld.MUL(result, op[0], op[1]); + break; + case nir_op_imul: assert(nir_dest_bit_size(instr->dest.dest) < 64); bld.MUL(result, op[0], op[1]); |