diff options
| -rw-r--r-- | src/compiler/glsl/ir_optimization.h | 1 | ||||
| -rw-r--r-- | src/compiler/glsl/lower_instructions.cpp | 149 |
2 files changed, 150 insertions, 0 deletions
diff --git a/src/compiler/glsl/ir_optimization.h b/src/compiler/glsl/ir_optimization.h index 2c7e6155ecb..c29260a45ca 100644 --- a/src/compiler/glsl/ir_optimization.h +++ b/src/compiler/glsl/ir_optimization.h @@ -48,6 +48,7 @@ #define REVERSE_TO_SHIFTS 0x10000 #define FIND_LSB_TO_FLOAT_CAST 0x20000 #define FIND_MSB_TO_FLOAT_CAST 0x40000 +#define IMUL_HIGH_TO_MUL 0x80000 /** * \see class lower_packing_builtins_visitor diff --git a/src/compiler/glsl/lower_instructions.cpp b/src/compiler/glsl/lower_instructions.cpp index 4be60273e0d..a9720f21638 100644 --- a/src/compiler/glsl/lower_instructions.cpp +++ b/src/compiler/glsl/lower_instructions.cpp @@ -165,6 +165,7 @@ private: void reverse_to_shifts(ir_expression *ir); void find_lsb_to_float_cast(ir_expression *ir); void find_msb_to_float_cast(ir_expression *ir); + void imul_high_to_mul(ir_expression *ir); }; } /* anonymous namespace */ @@ -1412,6 +1413,149 @@ lower_instructions_visitor::find_msb_to_float_cast(ir_expression *ir) this->progress = true; } +void +lower_instructions_visitor::imul_high_to_mul(ir_expression *ir) +{ + /* ABCD + * * EFGH + * ====== + * (GH * CD) + (GH * AB) << 16 + (EF * CD) << 16 + (EF * AB) << 32 + * + * In GLSL, (a * b) becomes + * + * uint m1 = (a & 0x0000ffffu) * (b & 0x0000ffffu); + * uint m2 = (a & 0x0000ffffu) * (b >> 16); + * uint m3 = (a >> 16) * (b & 0x0000ffffu); + * uint m4 = (a >> 16) * (b >> 16); + * + * uint c1; + * uint c2; + * uint lo_result; + * uint hi_result; + * + * lo_result = uaddCarry(m1, m2 << 16, c1); + * hi_result = m4 + c1; + * lo_result = uaddCarry(lo_result, m3 << 16, c2); + * hi_result = hi_result + c2; + * hi_result = hi_result + (m2 >> 16) + (m3 >> 16); + */ + const unsigned elements = ir->operands[0]->type->vector_elements; + ir_variable *src1 = + new(ir) ir_variable(glsl_type::uvec(elements), "src1", ir_var_temporary); + ir_variable *src1h = + new(ir) ir_variable(glsl_type::uvec(elements), "src1h", ir_var_temporary); + ir_variable *src1l = + new(ir) ir_variable(glsl_type::uvec(elements), "src1l", ir_var_temporary); + ir_variable *src2 = + new(ir) ir_variable(glsl_type::uvec(elements), "src2", ir_var_temporary); + ir_variable *src2h = + new(ir) ir_variable(glsl_type::uvec(elements), "src2h", ir_var_temporary); + ir_variable *src2l = + new(ir) ir_variable(glsl_type::uvec(elements), "src2l", ir_var_temporary); + ir_variable *t1 = + new(ir) ir_variable(glsl_type::uvec(elements), "t1", ir_var_temporary); + ir_variable *t2 = + new(ir) ir_variable(glsl_type::uvec(elements), "t2", ir_var_temporary); + ir_variable *lo = + new(ir) ir_variable(glsl_type::uvec(elements), "lo", ir_var_temporary); + ir_variable *hi = + new(ir) ir_variable(glsl_type::uvec(elements), "hi", ir_var_temporary); + ir_variable *different_signs = NULL; + ir_constant *c0000FFFF = new(ir) ir_constant(0x0000FFFFu, elements); + ir_constant *c16 = new(ir) ir_constant(16u, elements); + + ir_instruction &i = *base_ir; + + i.insert_before(src1); + i.insert_before(src2); + i.insert_before(src1h); + i.insert_before(src2h); + i.insert_before(src1l); + i.insert_before(src2l); + + if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) { + i.insert_before(assign(src1, ir->operands[0])); + i.insert_before(assign(src2, ir->operands[1])); + } else { + assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT); + + ir_variable *itmp1 = + new(ir) ir_variable(glsl_type::ivec(elements), "itmp1", ir_var_temporary); + ir_variable *itmp2 = + new(ir) ir_variable(glsl_type::ivec(elements), "itmp2", ir_var_temporary); + ir_constant *c0 = new(ir) ir_constant(int(0), elements); + + i.insert_before(itmp1); + i.insert_before(itmp2); + i.insert_before(assign(itmp1, ir->operands[0])); + i.insert_before(assign(itmp2, ir->operands[1])); + + different_signs = + new(ir) ir_variable(glsl_type::bvec(elements), "different_signs", + ir_var_temporary); + + i.insert_before(different_signs); + i.insert_before(assign(different_signs, expr(ir_binop_logic_xor, + less(itmp1, c0), + less(itmp2, c0->clone(ir, NULL))))); + + i.insert_before(assign(src1, i2u(abs(itmp1)))); + i.insert_before(assign(src2, i2u(abs(itmp2)))); + } + + i.insert_before(assign(src1l, bit_and(src1, c0000FFFF))); + i.insert_before(assign(src2l, bit_and(src2, c0000FFFF->clone(ir, NULL)))); + i.insert_before(assign(src1h, rshift(src1, c16))); + i.insert_before(assign(src2h, rshift(src2, c16->clone(ir, NULL)))); + + i.insert_before(lo); + i.insert_before(hi); + i.insert_before(t1); + i.insert_before(t2); + + i.insert_before(assign(lo, mul(src1l, src2l))); + i.insert_before(assign(t1, mul(src1l, src2h))); + i.insert_before(assign(t2, mul(src1h, src2l))); + i.insert_before(assign(hi, mul(src1h, src2h))); + + i.insert_before(assign(hi, add(hi, carry(lo, lshift(t1, c16->clone(ir, NULL)))))); + i.insert_before(assign(lo, add(lo, lshift(t1, c16->clone(ir, NULL))))); + + i.insert_before(assign(hi, add(hi, carry(lo, lshift(t2, c16->clone(ir, NULL)))))); + i.insert_before(assign(lo, add(lo, lshift(t2, c16->clone(ir, NULL))))); + + if (different_signs == NULL) { + assert(ir->operands[0]->type->base_type == GLSL_TYPE_UINT); + + ir->operation = ir_binop_add; + ir->operands[0] = add(hi, rshift(t1, c16->clone(ir, NULL))); + ir->operands[1] = rshift(t2, c16->clone(ir, NULL)); + } else { + assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT); + + i.insert_before(assign(hi, add(add(hi, rshift(t1, c16->clone(ir, NULL))), + rshift(t2, c16->clone(ir, NULL))))); + + /* For channels where different_signs is set we have to perform a 64-bit + * negation. This is *not* the same as just negating the high 32-bits. + * Consider -3 * 2. The high 32-bits is 0, but the desired result is + * -1, not -0! Recall -x == ~x + 1. + */ + ir_variable *neg_hi = + new(ir) ir_variable(glsl_type::ivec(elements), "neg_hi", ir_var_temporary); + ir_constant *c1 = new(ir) ir_constant(1u, elements); + + i.insert_before(neg_hi); + i.insert_before(assign(neg_hi, add(bit_not(u2i(hi)), + u2i(carry(bit_not(lo), c1))))); + + ir->operation = ir_triop_csel; + ir->operands[0] = new(ir) ir_dereference_variable(different_signs); + ir->operands[1] = new(ir) ir_dereference_variable(neg_hi); + ir->operands[2] = u2i(hi); + } +} + ir_visitor_status lower_instructions_visitor::visit_leave(ir_expression *ir) { @@ -1544,6 +1688,11 @@ lower_instructions_visitor::visit_leave(ir_expression *ir) find_msb_to_float_cast(ir); break; + case ir_binop_imul_high: + if (lowering(IMUL_HIGH_TO_MUL)) + imul_high_to_mul(ir); + break; + default: return visit_continue; } |