i965: Generate code for ir_binop_carry and ir_binop_borrow.

Using the ADDC and SUBB instructions on Gen7.

Reviewed-by: Kenneth Graunke <[email protected]>

14 files changed, 80 insertions, 0 deletions

diff --git a/src/mesa/drivers/dri/i965/brw_defines.h b/src/mesa/drivers/dri/i965/brw_defines.h
index 7dfb2b9136a..c1e7f31fe8b 100644
--- a/src/mesa/drivers/dri/i965/brw_defines.h
+++ b/src/mesa/drivers/dri/i965/brw_defines.h
@@ -733,6 +733,8 @@ enum opcode {
    BRW_OPCODE_FBH =	75,
    BRW_OPCODE_FBL =	76,
    BRW_OPCODE_CBIT =	77,
+   BRW_OPCODE_ADDC =	78,
+   BRW_OPCODE_SUBB =	79,
    BRW_OPCODE_SAD2 =	80,
    BRW_OPCODE_SADA2 =	81,
    BRW_OPCODE_DP4 =	84,
diff --git a/src/mesa/drivers/dri/i965/brw_disasm.c b/src/mesa/drivers/dri/i965/brw_disasm.c
index 18e6c9d78fc..22b37d797bf 100644
--- a/src/mesa/drivers/dri/i965/brw_disasm.c
+++ b/src/mesa/drivers/dri/i965/brw_disasm.c
@@ -77,6 +77,8 @@ const struct opcode_desc opcode_descs[128] = {
     [BRW_OPCODE_BFE] = { .name = "bfe", .nsrc = 3, .ndst = 1},
     [BRW_OPCODE_BFI1] = { .name = "bfe1", .nsrc = 2, .ndst = 1},
     [BRW_OPCODE_BFI2] = { .name = "bfe2", .nsrc = 3, .ndst = 1},
+    [BRW_OPCODE_ADDC] = { .name = "addc", .nsrc = 2, .ndst = 1},
+    [BRW_OPCODE_SUBB] = { .name = "subb", .nsrc = 2, .ndst = 1},
 
     [BRW_OPCODE_SEND] = { .name = "send", .nsrc = 1, .ndst = 1 },
     [BRW_OPCODE_SENDC] = { .name = "sendc", .nsrc = 1, .ndst = 1 },
diff --git a/src/mesa/drivers/dri/i965/brw_eu.h b/src/mesa/drivers/dri/i965/brw_eu.h
index 720bc74591a..072310d5544 100644
--- a/src/mesa/drivers/dri/i965/brw_eu.h
+++ b/src/mesa/drivers/dri/i965/brw_eu.h
@@ -181,6 +181,8 @@ ALU3(BFI2)
 ALU1(FBH)
 ALU1(FBL)
 ALU1(CBIT)
+ALU2(ADDC)
+ALU2(SUBB)
 
 ROUND(RNDZ)
 ROUND(RNDE)
diff --git a/src/mesa/drivers/dri/i965/brw_eu_emit.c b/src/mesa/drivers/dri/i965/brw_eu_emit.c
index 7ed3df0e5f8..d9ba110f27f 100644
--- a/src/mesa/drivers/dri/i965/brw_eu_emit.c
+++ b/src/mesa/drivers/dri/i965/brw_eu_emit.c
@@ -968,6 +968,8 @@ ALU3(BFI2)
 ALU1(FBH)
 ALU1(FBL)
 ALU1(CBIT)
+ALU2(ADDC)
+ALU2(SUBB)
 
 ROUND(RNDZ)
 ROUND(RNDE)
diff --git a/src/mesa/drivers/dri/i965/brw_fs.cpp b/src/mesa/drivers/dri/i965/brw_fs.cpp
index 89eb33e480d..b2c9f5a8e32 100644
--- a/src/mesa/drivers/dri/i965/brw_fs.cpp
+++ b/src/mesa/drivers/dri/i965/brw_fs.cpp
@@ -180,6 +180,8 @@ ALU1(FBH)
 ALU1(FBL)
 ALU1(CBIT)
 ALU3(MAD)
+ALU2(ADDC)
+ALU2(SUBB)
 
 /** Gen4 predicated IF. */
 fs_inst *
diff --git a/src/mesa/drivers/dri/i965/brw_fs.h b/src/mesa/drivers/dri/i965/brw_fs.h
index 679a4b0dc1b..c59ca51a1f9 100644
--- a/src/mesa/drivers/dri/i965/brw_fs.h
+++ b/src/mesa/drivers/dri/i965/brw_fs.h
@@ -260,6 +260,8 @@ public:
    fs_inst *FBL(fs_reg dst, fs_reg value);
    fs_inst *CBIT(fs_reg dst, fs_reg value);
    fs_inst *MAD(fs_reg dst, fs_reg c, fs_reg b, fs_reg a);
+   fs_inst *ADDC(fs_reg dst, fs_reg src0, fs_reg src1);
+   fs_inst *SUBB(fs_reg dst, fs_reg src0, fs_reg src1);
 
    int type_size(const struct glsl_type *type);
    fs_inst *get_instruction_generating_reg(fs_inst *start,
diff --git a/src/mesa/drivers/dri/i965/brw_fs_channel_expressions.cpp b/src/mesa/drivers/dri/i965/brw_fs_channel_expressions.cpp
index fb932d8c217..8cbe1c9bc99 100644
--- a/src/mesa/drivers/dri/i965/brw_fs_channel_expressions.cpp
+++ b/src/mesa/drivers/dri/i965/brw_fs_channel_expressions.cpp
@@ -234,6 +234,8 @@ ir_channel_expressions_visitor::visit_leave(ir_assignment *ir)
    case ir_binop_sub:
    case ir_binop_mul:
    case ir_binop_div:
+   case ir_binop_carry:
+   case ir_binop_borrow:
    case ir_binop_mod:
    case ir_binop_min:
    case ir_binop_max:
diff --git a/src/mesa/drivers/dri/i965/brw_fs_copy_propagation.cpp b/src/mesa/drivers/dri/i965/brw_fs_copy_propagation.cpp
index 034ebef9917..5acef207af3 100644
--- a/src/mesa/drivers/dri/i965/brw_fs_copy_propagation.cpp
+++ b/src/mesa/drivers/dri/i965/brw_fs_copy_propagation.cpp
@@ -343,6 +343,8 @@ fs_visitor::try_constant_propagate(fs_inst *inst, acp_entry *entry)
 
       case BRW_OPCODE_SHL:
       case BRW_OPCODE_SHR:
+      case BRW_OPCODE_ADDC:
+      case BRW_OPCODE_SUBB:
          if (i == 1) {
             inst->src[i] = entry->src;
             progress = true;
diff --git a/src/mesa/drivers/dri/i965/brw_fs_generator.cpp b/src/mesa/drivers/dri/i965/brw_fs_generator.cpp
index 1e27152a67d..8fe42033481 100644
--- a/src/mesa/drivers/dri/i965/brw_fs_generator.cpp
+++ b/src/mesa/drivers/dri/i965/brw_fs_generator.cpp
@@ -1334,6 +1334,18 @@ fs_generator::generate_code(exec_list *instructions)
          /* CBIT only supports UD type for dst. */
          brw_CBIT(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
          break;
+      case BRW_OPCODE_ADDC:
+         assert(brw->gen >= 7);
+         brw_set_acc_write_control(p, 1);
+         brw_ADDC(p, dst, src[0], src[1]);
+         brw_set_acc_write_control(p, 0);
+         break;
+      case BRW_OPCODE_SUBB:
+         assert(brw->gen >= 7);
+         brw_set_acc_write_control(p, 1);
+         brw_SUBB(p, dst, src[0], src[1]);
+         brw_set_acc_write_control(p, 0);
+         break;
 
       case BRW_OPCODE_BFE:
          brw_set_access_mode(p, BRW_ALIGN_16);
diff --git a/src/mesa/drivers/dri/i965/brw_fs_visitor.cpp b/src/mesa/drivers/dri/i965/brw_fs_visitor.cpp
index 62d2042a598..f35e1127646 100644
--- a/src/mesa/drivers/dri/i965/brw_fs_visitor.cpp
+++ b/src/mesa/drivers/dri/i965/brw_fs_visitor.cpp
@@ -447,6 +447,26 @@ fs_visitor::visit(ir_expression *ir)
       assert(ir->type->is_integer());
       emit_math(SHADER_OPCODE_INT_QUOTIENT, this->result, op[0], op[1]);
       break;
+   case ir_binop_carry: {
+      if (brw->gen >= 7 && dispatch_width == 16)
+         fail("16-wide explicit accumulator operands unsupported\n");
+
+      struct brw_reg acc = retype(brw_acc_reg(), BRW_REGISTER_TYPE_UD);
+
+      emit(ADDC(reg_null_ud, op[0], op[1]));
+      emit(MOV(this->result, fs_reg(acc)));
+      break;
+   }
+   case ir_binop_borrow: {
+      if (brw->gen >= 7 && dispatch_width == 16)
+         fail("16-wide explicit accumulator operands unsupported\n");
+
+      struct brw_reg acc = retype(brw_acc_reg(), BRW_REGISTER_TYPE_UD);
+
+      emit(SUBB(reg_null_ud, op[0], op[1]));
+      emit(MOV(this->result, fs_reg(acc)));
+      break;
+   }
    case ir_binop_mod:
       /* Floating point should be lowered by MOD_TO_FRACT in the compiler. */
       assert(ir->type->is_integer());
diff --git a/src/mesa/drivers/dri/i965/brw_vec4.h b/src/mesa/drivers/dri/i965/brw_vec4.h
index 9683f1c06cd..290363e416b 100644
--- a/src/mesa/drivers/dri/i965/brw_vec4.h
+++ b/src/mesa/drivers/dri/i965/brw_vec4.h
@@ -419,6 +419,8 @@ public:
    vec4_instruction *FBL(dst_reg dst, src_reg value);
    vec4_instruction *CBIT(dst_reg dst, src_reg value);
    vec4_instruction *MAD(dst_reg dst, src_reg c, src_reg b, src_reg a);
+   vec4_instruction *ADDC(dst_reg dst, src_reg src0, src_reg src1);
+   vec4_instruction *SUBB(dst_reg dst, src_reg src0, src_reg src1);
 
    int implied_mrf_writes(vec4_instruction *inst);
 
diff --git a/src/mesa/drivers/dri/i965/brw_vec4_copy_propagation.cpp b/src/mesa/drivers/dri/i965/brw_vec4_copy_propagation.cpp
index cd2b11831ae..d009a08e8bd 100644
--- a/src/mesa/drivers/dri/i965/brw_vec4_copy_propagation.cpp
+++ b/src/mesa/drivers/dri/i965/brw_vec4_copy_propagation.cpp
@@ -97,6 +97,8 @@ try_constant_propagation(vec4_instruction *inst, int arg, src_reg *values[4])
 
    case BRW_OPCODE_SHL:
    case BRW_OPCODE_SHR:
+   case BRW_OPCODE_ADDC:
+   case BRW_OPCODE_SUBB:
       if (arg == 1) {
          inst->src[arg] = value;
          return true;
diff --git a/src/mesa/drivers/dri/i965/brw_vec4_generator.cpp b/src/mesa/drivers/dri/i965/brw_vec4_generator.cpp
index 00efb10f38b..ae6e43ca34a 100644
--- a/src/mesa/drivers/dri/i965/brw_vec4_generator.cpp
+++ b/src/mesa/drivers/dri/i965/brw_vec4_generator.cpp
@@ -963,6 +963,18 @@ vec4_generator::generate_vec4_instruction(vec4_instruction *instruction,
       /* CBIT only supports UD type for dst. */
       brw_CBIT(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
       break;
+   case BRW_OPCODE_ADDC:
+      assert(brw->gen >= 7);
+      brw_set_acc_write_control(p, 1);
+      brw_ADDC(p, dst, src[0], src[1]);
+      brw_set_acc_write_control(p, 0);
+      break;
+   case BRW_OPCODE_SUBB:
+      assert(brw->gen >= 7);
+      brw_set_acc_write_control(p, 1);
+      brw_SUBB(p, dst, src[0], src[1]);
+      brw_set_acc_write_control(p, 0);
+      break;
 
    case BRW_OPCODE_BFE:
       brw_BFE(p, dst, src[0], src[1], src[2]);
diff --git a/src/mesa/drivers/dri/i965/brw_vec4_visitor.cpp b/src/mesa/drivers/dri/i965/brw_vec4_visitor.cpp
index 09f0236c682..856312f4dd9 100644
--- a/src/mesa/drivers/dri/i965/brw_vec4_visitor.cpp
+++ b/src/mesa/drivers/dri/i965/brw_vec4_visitor.cpp
@@ -162,6 +162,8 @@ ALU1(FBH)
 ALU1(FBL)
 ALU1(CBIT)
 ALU3(MAD)
+ALU2(ADDC)
+ALU2(SUBB)
 
 /** Gen4 predicated IF. */
 vec4_instruction *
@@ -1371,6 +1373,20 @@ vec4_visitor::visit(ir_expression *ir)
       assert(ir->type->is_integer());
       emit_math(SHADER_OPCODE_INT_QUOTIENT, result_dst, op[0], op[1]);
       break;
+   case ir_binop_carry: {
+      struct brw_reg acc = retype(brw_acc_reg(), BRW_REGISTER_TYPE_UD);
+
+      emit(ADDC(dst_null_ud(), op[0], op[1]));
+      emit(MOV(result_dst, src_reg(acc)));
+      break;
+   }
+   case ir_binop_borrow: {
+      struct brw_reg acc = retype(brw_acc_reg(), BRW_REGISTER_TYPE_UD);
+
+      emit(SUBB(dst_null_ud(), op[0], op[1]));
+      emit(MOV(result_dst, src_reg(acc)));
+      break;
+   }
    case ir_binop_mod:
       /* Floating point should be lowered by MOD_TO_FRACT in the compiler. */
       assert(ir->type->is_integer());