pan/midgard: Add bitwise src/invert fusing

De Morgan's Laws and some special ops basically.

Signed-off-by: Alyssa Rosenzweig <[email protected]>

3 files changed, 124 insertions, 0 deletions

diff --git a/src/panfrost/midgard/compiler.h b/src/panfrost/midgard/compiler.h
index cf9db9145a3..2aad68f14dd 100644
--- a/src/panfrost/midgard/compiler.h
+++ b/src/panfrost/midgard/compiler.h
@@ -577,6 +577,7 @@ void midgard_opt_post_move_eliminate(compiler_context *ctx, midgard_block *block
 
 void midgard_lower_invert(compiler_context *ctx, midgard_block *block);
 bool midgard_opt_not_propagate(compiler_context *ctx, midgard_block *block);
+bool midgard_opt_fuse_src_invert(compiler_context *ctx, midgard_block *block);
 bool midgard_opt_fuse_dest_invert(compiler_context *ctx, midgard_block *block);
 
 #endif
diff --git a/src/panfrost/midgard/midgard_compile.c b/src/panfrost/midgard/midgard_compile.c
index a3c66ca43bb..c4a50cbaee0 100644
--- a/src/panfrost/midgard/midgard_compile.c
+++ b/src/panfrost/midgard/midgard_compile.c
@@ -2358,6 +2358,7 @@ midgard_compile_shader_nir(struct midgard_screen *screen, nir_shader *nir, midga
                         progress |= midgard_opt_combine_projection(ctx, block);
                         progress |= midgard_opt_varying_projection(ctx, block);
                         progress |= midgard_opt_not_propagate(ctx, block);
+                        progress |= midgard_opt_fuse_src_invert(ctx, block);
                         progress |= midgard_opt_fuse_dest_invert(ctx, block);
                 }
         } while (progress);
diff --git a/src/panfrost/midgard/midgard_opt_invert.c b/src/panfrost/midgard/midgard_opt_invert.c
index ffe43a1b176..6a6be1d83ff 100644
--- a/src/panfrost/midgard/midgard_opt_invert.c
+++ b/src/panfrost/midgard/midgard_opt_invert.c
@@ -132,6 +132,32 @@ mir_invert_op(midgard_alu_op op)
         }
 }
 
+static midgard_alu_op
+mir_demorgan_op(midgard_alu_op op)
+{
+        switch (op) {
+        case midgard_alu_op_iand:
+                return midgard_alu_op_inor;
+        case midgard_alu_op_ior:
+                return midgard_alu_op_inand;
+        default:
+                unreachable("Op not De Morgan-able");
+        }
+}
+
+static midgard_alu_op
+mir_notright_op(midgard_alu_op op)
+{
+        switch (op) {
+        case midgard_alu_op_iand:
+                return midgard_alu_op_iandnot;
+        case midgard_alu_op_ior:
+                return midgard_alu_op_iornot;
+        default:
+                unreachable("Op not right able");
+        }
+}
+
 bool
 midgard_opt_fuse_dest_invert(compiler_context *ctx, midgard_block *block)
 {
@@ -150,3 +176,99 @@ midgard_opt_fuse_dest_invert(compiler_context *ctx, midgard_block *block)
 
         return progress;
 }
+
+/* Next up, we can fuse inverts into the sources of bitwise ops:
+ *
+ * ~a & b = b & ~a = iandnot(b, a)
+ * a & ~b = iandnot(a, b)
+ * ~a & ~b = ~(a | b) = inor(a, b)
+ *
+ * ~a | b = b | ~a = iornot(b, a)
+ *  a | ~b = iornot(a, b)
+ * ~a | ~b = ~(a & b) = inand(a, b)
+ *
+ * ~a ^ b = ~(a ^ b) = inxor(a, b)
+ * a ^ ~b = ~(a ^ b) + inxor(a, b)
+ * ~a ^ ~b = a ^ b
+ * ~(a ^ b) = inxor(a, b)
+ */
+
+static bool
+mir_strip_inverted(compiler_context *ctx, unsigned node)
+{
+       /* Strips and returns the invert off a node */
+       mir_foreach_instr_global(ctx, ins) {
+               if (ins->compact_branch) continue;
+               if (ins->ssa_args.dest != node) continue;
+
+               bool status = ins->invert;
+               ins->invert = false;
+               return status;
+       }
+
+       unreachable("Invalid node stripped");
+}
+
+bool
+midgard_opt_fuse_src_invert(compiler_context *ctx, midgard_block *block)
+{
+        bool progress = false;
+
+        mir_foreach_instr_in_block_safe(block, ins) {
+                /* Search for inverted bitwise */
+                if (ins->type != TAG_ALU_4) continue;
+                if (!mir_is_bitwise(ins)) continue;
+                if (ins->invert) continue;
+
+                if (ins->ssa_args.src0 & IS_REG) continue;
+                if (ins->ssa_args.src1 & IS_REG) continue;
+                if (!mir_single_use(ctx, ins->ssa_args.src0)) continue;
+                if (!ins->ssa_args.inline_constant && !mir_single_use(ctx, ins->ssa_args.src1)) continue;
+
+                bool not_a = mir_strip_inverted(ctx, ins->ssa_args.src0);
+                bool not_b =
+                        ins->ssa_args.inline_constant ? false :
+                        mir_strip_inverted(ctx, ins->ssa_args.src1);
+
+                /* Edge case: if src0 == src1, it'll've been stripped */
+                if ((ins->ssa_args.src0 == ins->ssa_args.src1) && !ins->ssa_args.inline_constant)
+                        not_b = not_a;
+
+                progress |= (not_a || not_b);
+
+                /* No point */
+                if (!(not_a || not_b)) continue;
+
+                bool both = not_a && not_b;
+                bool left = not_a && !not_b;
+                bool right = !not_a && not_b;
+
+                /* No-op, but we got to strip the inverts */
+                if (both && ins->alu.op == midgard_alu_op_ixor)
+                        continue;
+
+                if (both) {
+                        ins->alu.op = mir_demorgan_op(ins->alu.op);
+                } else if (right || (left && !ins->ssa_args.inline_constant)) {
+                        if (left) {
+                                /* Commute */
+                                unsigned temp = ins->ssa_args.src0;
+                                ins->ssa_args.src0 = ins->ssa_args.src1;
+                                ins->ssa_args.src1 = temp;
+                        }
+
+                        ins->alu.op = mir_notright_op(ins->alu.op);
+                } else if (left && ins->ssa_args.inline_constant) {
+                        /* Some special transformations:
+                         *
+                         * ~A & c = ~(~(~A) | (~c)) = ~(A | ~c) = inor(A, ~c)
+                         * ~A | c = ~(~(~A) & (~c)) = ~(A & ~c) = inand(A, ~c)
+                         */
+
+                        ins->alu.op = mir_demorgan_op(ins->alu.op);
+                        ins->inline_constant = ~ins->inline_constant;
+                }
+        }
+
+        return progress;
+}