nir/algebraic: Add the ability to mark a replacement as exact

Reviewed-by: Matt Turner <[email protected]>
Reviewed-by: Connor Abbott <[email protected]>

4 files changed, 13 insertions, 3 deletions

diff --git a/src/compiler/nir/nir_algebraic.py b/src/compiler/nir/nir_algebraic.py
index e13872869a5..480dcaf3cf5 100644
--- a/src/compiler/nir/nir_algebraic.py
+++ b/src/compiler/nir/nir_algebraic.py
@@ -200,7 +200,7 @@ class Value(object):
    ${val.cond if val.cond else 'NULL'},
    ${val.swizzle()},
 % elif isinstance(val, Expression):
-   ${'true' if val.inexact else 'false'},
+   ${'true' if val.inexact else 'false'}, ${'true' if val.exact else 'false'},
    ${val.comm_expr_idx}, ${val.comm_exprs},
    ${val.c_opcode()},
    { ${', '.join(src.c_value_ptr(cache) for src in val.sources)} },
@@ -348,7 +348,7 @@ class Variable(Value):
          return '{' + ', '.join([str(swizzles[c]) for c in self.swiz[1:]]) + '}'
       return '{0, 1, 2, 3}'
 
-_opcode_re = re.compile(r"(?P<inexact>~)?(?P<opcode>\w+)(?:@(?P<bits>\d+))?"
+_opcode_re = re.compile(r"(?P<inexact>~)?(?P<exact>!)?(?P<opcode>\w+)(?:@(?P<bits>\d+))?"
                         r"(?P<cond>\([^\)]+\))?")
 
 class Expression(Value):
@@ -362,8 +362,12 @@ class Expression(Value):
       self.opcode = m.group('opcode')
       self._bit_size = int(m.group('bits')) if m.group('bits') else None
       self.inexact = m.group('inexact') is not None
+      self.exact = m.group('exact') is not None
       self.cond = m.group('cond')
 
+      assert not self.inexact or not self.exact, \
+            'Expression cannot be both exact and inexact.'
+
       # "many-comm-expr" isn't really a condition.  It's notification to the
       # generator that this pattern is known to have too many commutative
       # expressions, and an error should not be generated for this case.
diff --git a/src/compiler/nir/nir_opt_algebraic.py b/src/compiler/nir/nir_opt_algebraic.py
index aad0d0056e1..c706eed5258 100644
--- a/src/compiler/nir/nir_opt_algebraic.py
+++ b/src/compiler/nir/nir_opt_algebraic.py
@@ -69,6 +69,9 @@ e = 'e'
 # expression this indicates that the constructed value should have that
 # bit-size.
 #
+# If the opcode in a replacement expression is prefixed by a '!' character,
+# this indicated that the new expression will be marked exact.
+#
 # A special condition "many-comm-expr" can be used with expressions to note
 # that the expression and its subexpressions have more commutative expressions
 # than nir_replace_instr can handle.  If this special condition is needed with
diff --git a/src/compiler/nir/nir_search.c b/src/compiler/nir/nir_search.c
index db2a606c7b6..b78d3046a7b 100644
--- a/src/compiler/nir/nir_search.c
+++ b/src/compiler/nir/nir_search.c
@@ -474,7 +474,7 @@ construct_value(nir_builder *build,
        * expression we are replacing has any exact values, the entire
        * replacement should be exact.
        */
-      alu->exact = state->has_exact_alu;
+      alu->exact = state->has_exact_alu || expr->exact;
 
       for (unsigned i = 0; i < nir_op_infos[op].num_inputs; i++) {
          /* If the source is an explicitly sized source, then we need to reset
diff --git a/src/compiler/nir/nir_search.h b/src/compiler/nir/nir_search.h
index e5c29f3e94a..80d153916c8 100644
--- a/src/compiler/nir/nir_search.h
+++ b/src/compiler/nir/nir_search.h
@@ -138,6 +138,9 @@ typedef struct {
     */
    bool inexact;
 
+   /** In a replacement, requests that the instruction be marked exact. */
+   bool exact;
+
    /* Commutative expression index.  This is assigned by opt_algebraic.py when
     * search structures are constructed and is a unique (to this structure)
     * index within the commutative operation bitfield used for searching for