pan/midgard: Introduce invert field

This will enable us to fuse inverts in various ways. Marginal hurt:

total instructions in shared programs: 3610 -> 3611 (0.03%)
instructions in affected programs: 67 -> 68 (1.49%)
helped: 0
HURT: 1

Signed-off-by: Alyssa Rosenzweig <[email protected]>

5 files changed, 90 insertions, 11 deletions

diff --git a/src/panfrost/midgard/compiler.h b/src/panfrost/midgard/compiler.h
index f0cdc03038b..a7698941f4e 100644
--- a/src/panfrost/midgard/compiler.h
+++ b/src/panfrost/midgard/compiler.h
@@ -123,8 +123,15 @@ typedef struct midgard_instruction {
         /* Masks in a saneish format. One bit per channel, not packed fancy.
          * Use this instead of the op specific ones, and switch over at emit
          * time */
+
         uint16_t mask;
 
+        /* For ALU ops only: set to true to invert (bitwise NOT) the
+         * destination of an integer-out op. Not imeplemented in hardware but
+         * allows more optimizations */
+
+        bool invert;
+
         union {
                 midgard_load_store_word load_store;
                 midgard_vector_alu alu;
@@ -524,4 +531,6 @@ bool midgard_opt_dead_code_eliminate(compiler_context *ctx, midgard_block *block
 bool midgard_opt_dead_move_eliminate(compiler_context *ctx, midgard_block *block);
 void midgard_opt_post_move_eliminate(compiler_context *ctx, midgard_block *block, struct ra_graph *g);
 
+void midgard_lower_invert(compiler_context *ctx, midgard_block *block);
+
 #endif
diff --git a/src/panfrost/midgard/meson.build b/src/panfrost/midgard/meson.build
index 670d5ef075a..55d8d6df0cc 100644
--- a/src/panfrost/midgard/meson.build
+++ b/src/panfrost/midgard/meson.build
@@ -32,6 +32,7 @@ libpanfrost_midgard_files = files(
   'mir_promote_uniforms.c',
   'midgard_opt_copy_prop.c',
   'midgard_opt_dce.c',
+  'midgard_opt_invert.c',
   'midgard_opt_perspective.c',
   'cppwrap.cpp',
   'disassemble.c',
diff --git a/src/panfrost/midgard/midgard_compile.c b/src/panfrost/midgard/midgard_compile.c
index 46f0725741b..646646423c0 100644
--- a/src/panfrost/midgard/midgard_compile.c
+++ b/src/panfrost/midgard/midgard_compile.c
@@ -304,12 +304,6 @@ nir_dest_index(compiler_context *ctx, nir_dest *dst)
         }
 }
 
-static unsigned
-make_compiler_temp(compiler_context *ctx)
-{
-        return ctx->func->impl->ssa_alloc + ctx->func->impl->reg_alloc + ctx->temp_alloc++;
-}
-
 static int sysval_for_instr(compiler_context *ctx, nir_instr *instr,
                             unsigned *dest)
 {
@@ -859,8 +853,8 @@ emit_alu(compiler_context *ctx, nir_alu_instr *instr)
                 ALU_CASE(fsin, fsin);
                 ALU_CASE(fcos, fcos);
 
-                /* Second op implicit #0 */
-                ALU_CASE(inot, inor);
+                /* We'll set invert */
+                ALU_CASE(inot, imov);
                 ALU_CASE(iand, iand);
                 ALU_CASE(ior, ior);
                 ALU_CASE(ixor, ixor);
@@ -1120,9 +1114,7 @@ emit_alu(compiler_context *ctx, nir_alu_instr *instr)
                 ins.constants[0] = 0.0f;
                 ins.alu.src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx);
         } else if (instr->op == nir_op_inot) {
-                /* ~b = ~(b & b), so duplicate the source */
-                ins.ssa_args.src1 = ins.ssa_args.src0;
-                ins.alu.src2 = ins.alu.src1;
+                ins.invert = true;
         }
 
         if ((opcode_props & UNITS_ALL) == UNIT_VLUT) {
@@ -2380,6 +2372,10 @@ midgard_compile_shader_nir(struct midgard_screen *screen, nir_shader *nir, midga
                 }
         } while (progress);
 
+        mir_foreach_block(ctx, block) {
+                midgard_lower_invert(ctx, block);
+        }
+
         /* Nested control-flow can result in dead branches at the end of the
          * block. This messes with our analysis and is just dead code, so cull
          * them */
@@ -2387,6 +2383,11 @@ midgard_compile_shader_nir(struct midgard_screen *screen, nir_shader *nir, midga
                 midgard_opt_cull_dead_branch(ctx, block);
         }
 
+        /* Ensure we were lowered */
+        mir_foreach_instr_global(ctx, ins) {
+                assert(!ins->invert);
+        }
+
         /* Schedule! */
         schedule_program(ctx);
 
diff --git a/src/panfrost/midgard/midgard_opt_invert.c b/src/panfrost/midgard/midgard_opt_invert.c
new file mode 100644
index 00000000000..f5c10981ccb
--- /dev/null
+++ b/src/panfrost/midgard/midgard_opt_invert.c
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2019 Collabora, Ltd.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "compiler.h"
+#include "midgard_ops.h"
+
+/* Lowers the invert field on instructions to a dedicated inot (inor)
+ * instruction instead, as invert is not always supported natively by the
+ * hardware */
+
+void
+midgard_lower_invert(compiler_context *ctx, midgard_block *block)
+{
+        mir_foreach_instr_in_block_safe(block, ins) {
+                if (ins->type != TAG_ALU_4) continue;
+                if (!ins->invert) continue;
+
+                unsigned temp = make_compiler_temp(ctx);
+
+                midgard_instruction not = {
+                        .type = TAG_ALU_4,
+                        .mask = ins->mask,
+                        .ssa_args = {
+                                .src0 = temp,
+                                .src1 = 0,
+                                .dest = ins->ssa_args.dest,
+                                .inline_constant = true
+                        },
+                        .alu = {
+                                .op = midgard_alu_op_inor,
+                                /* TODO: i16 */
+                                .reg_mode = midgard_reg_mode_32,
+                                .dest_override = midgard_dest_override_none,
+                                .outmod = midgard_outmod_int_wrap,
+                                .src1 = vector_alu_srco_unsigned(blank_alu_src),
+                                .src2 = vector_alu_srco_unsigned(zero_alu_src)
+                        },
+                };
+
+                ins->ssa_args.dest = temp;
+                ins->invert = false;
+                mir_insert_instruction_before(mir_next_op(ins), not);
+        }
+}
diff --git a/src/panfrost/midgard/mir.c b/src/panfrost/midgard/mir.c
index 963652eed3d..2d100728724 100644
--- a/src/panfrost/midgard/mir.c
+++ b/src/panfrost/midgard/mir.c
@@ -150,6 +150,10 @@ mir_nontrivial_outmod(midgard_instruction *ins)
         bool is_int = midgard_is_integer_op(ins->alu.op);
         unsigned mod = ins->alu.outmod;
 
+        /* Pseudo-outmod */
+        if (ins->invert)
+                return true;
+
         /* Type conversion is a sort of outmod */
         if (ins->alu.dest_override != midgard_dest_override_none)
                 return true;