nir: Collect if shader uses cross-invocation or indirect I/O.

The following new fields are added to tess shader info:

* `tcs_cross_invocation_inputs_read`
* `tcs_cross_invocation_outputs_read`

These are I/O masks that are a subset of inputs_read and outputs_read
and they contain which per-vertex inputs and outputs are read
cross-invocation.

Additionall, the following new fields are added to shader_info:

* `inputs_read_indirectly`
* `outputs_accessed_indirectly`
* `patch_inputs_read_indirectly`
* `patch_outputs_accessed_indirectly`

These new fields can be used for optimizing TCS in a back-end compiler.
If you can be sure that the TCS doesn't use cross-invocation inputs
or outputs, you can choose a different strategy for storing VS and TCS
outputs. However, such optimizations might need to be disabled when
the inputs/outputs are accessed indirectly due to backend limitations,
so this information is also collected.

Example: RADV currently has to store all VS and TCS outputs in LDS, but
for shaders when only inputs and/or outputs belonging to the current
invocation ID are used, it could skip storing these in LDS entirely.

Signed-off-by: Timur Kristóf <[email protected]>
Reviewed-by: Rhys Perry <[email protected]>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/4165>

2 files changed, 110 insertions, 13 deletions

diff --git a/src/compiler/nir/nir_gather_info.c b/src/compiler/nir/nir_gather_info.c
index 44d3c523d09..d902fb7b69e 100644
--- a/src/compiler/nir/nir_gather_info.c
+++ b/src/compiler/nir/nir_gather_info.c
@@ -22,11 +22,54 @@
  */
 
 #include "nir.h"
+#include "nir_deref.h"
 #include "main/menums.h"
 
 static void
+get_deref_info(nir_shader *shader, nir_variable *var, nir_deref_instr *deref,
+               bool *cross_invocation, bool *indirect)
+{
+   *cross_invocation = false;
+   *indirect = false;
+
+   const bool per_vertex = nir_is_per_vertex_io(var, shader->info.stage);
+
+   nir_deref_path path;
+   nir_deref_path_init(&path, deref, NULL);
+   assert(path.path[0]->deref_type == nir_deref_type_var);
+   nir_deref_instr **p = &path.path[1];
+
+   /* Vertex index is the outermost array index. */
+   if (per_vertex) {
+      assert((*p)->deref_type == nir_deref_type_array);
+      nir_instr *vertex_index_instr = (*p)->arr.index.ssa->parent_instr;
+      *cross_invocation =
+         vertex_index_instr->type != nir_instr_type_intrinsic ||
+         nir_instr_as_intrinsic(vertex_index_instr)->intrinsic !=
+            nir_intrinsic_load_invocation_id;
+      p++;
+   }
+
+   /* We always lower indirect dereferences for "compact" array vars. */
+   if (!path.path[0]->var->data.compact) {
+      /* Non-compact array vars: find out if they are indirect. */
+      for (; *p; p++) {
+         if ((*p)->deref_type == nir_deref_type_array) {
+            *indirect |= !nir_src_is_const((*p)->arr.index);
+         } else if ((*p)->deref_type == nir_deref_type_struct) {
+            /* Struct indices are always constant. */
+         } else {
+            unreachable("Unsupported deref type");
+         }
+      }
+   }
+
+   nir_deref_path_finish(&path);
+}
+
+static void
 set_io_mask(nir_shader *shader, nir_variable *var, int offset, int len,
-            bool is_output_read)
+            nir_deref_instr *deref, bool is_output_read)
 {
    for (int i = 0; i < len; i++) {
       assert(var->data.location != -1);
@@ -48,11 +91,23 @@ set_io_mask(nir_shader *shader, nir_variable *var, int offset, int len,
          bitfield = BITFIELD64_BIT(idx);
       }
 
+      bool cross_invocation;
+      bool indirect;
+      get_deref_info(shader, var, deref, &cross_invocation, &indirect);
+
       if (var->data.mode == nir_var_shader_in) {
-         if (is_patch_generic)
+         if (is_patch_generic) {
             shader->info.patch_inputs_read |= bitfield;
-         else
+            if (indirect)
+               shader->info.patch_inputs_read_indirectly |= bitfield;
+         } else {
             shader->info.inputs_read |= bitfield;
+            if (indirect)
+               shader->info.inputs_read_indirectly |= bitfield;
+         }
+
+         if (cross_invocation)
+            shader->info.tess.tcs_cross_invocation_inputs_read |= bitfield;
 
          if (shader->info.stage == MESA_SHADER_FRAGMENT) {
             shader->info.fs.uses_sample_qualifier |= var->data.sample;
@@ -62,16 +117,27 @@ set_io_mask(nir_shader *shader, nir_variable *var, int offset, int len,
          if (is_output_read) {
             if (is_patch_generic) {
                shader->info.patch_outputs_read |= bitfield;
+               if (indirect)
+                  shader->info.patch_outputs_accessed_indirectly |= bitfield;
             } else {
                shader->info.outputs_read |= bitfield;
+               if (indirect)
+                  shader->info.outputs_accessed_indirectly |= bitfield;
             }
+
+            if (cross_invocation)
+               shader->info.tess.tcs_cross_invocation_outputs_read |= bitfield;
          } else {
-	    if (is_patch_generic) {
-	       shader->info.patch_outputs_written |= bitfield;
-	    } else if (!var->data.read_only) {
-	       shader->info.outputs_written |= bitfield;
-	    }
-	 }
+            if (is_patch_generic) {
+               shader->info.patch_outputs_written |= bitfield;
+               if (indirect)
+                  shader->info.patch_outputs_accessed_indirectly |= bitfield;
+            } else if (!var->data.read_only) {
+               shader->info.outputs_written |= bitfield;
+               if (indirect)
+                  shader->info.outputs_accessed_indirectly |= bitfield;
+            }
+         }
 
 
          if (var->data.fb_fetch_output)
@@ -85,7 +151,8 @@ set_io_mask(nir_shader *shader, nir_variable *var, int offset, int len,
  * represents a shader input or output.
  */
 static void
-mark_whole_variable(nir_shader *shader, nir_variable *var, bool is_output_read)
+mark_whole_variable(nir_shader *shader, nir_variable *var,
+                    nir_deref_instr *deref, bool is_output_read)
 {
    const struct glsl_type *type = var->type;
 
@@ -98,7 +165,7 @@ mark_whole_variable(nir_shader *shader, nir_variable *var, bool is_output_read)
       var->data.compact ? DIV_ROUND_UP(glsl_get_length(type), 4)
                         : glsl_count_attribute_slots(type, false);
 
-   set_io_mask(shader, var, 0, slots, is_output_read);
+   set_io_mask(shader, var, 0, slots, deref, is_output_read);
 }
 
 static unsigned
@@ -193,7 +260,7 @@ try_mask_partial_io(nir_shader *shader, nir_variable *var,
       return false;
    }
 
-   set_io_mask(shader, var, offset, elem_width, is_output_read);
+   set_io_mask(shader, var, offset, elem_width, deref, is_output_read);
    return true;
 }
 
@@ -228,7 +295,7 @@ gather_intrinsic_info(nir_intrinsic_instr *instr, nir_shader *shader,
             is_output_read = true;
 
          if (!try_mask_partial_io(shader, var, deref, is_output_read))
-            mark_whole_variable(shader, var, is_output_read);
+            mark_whole_variable(shader, var, deref, is_output_read);
 
          /* We need to track which input_reads bits correspond to a
           * dvec3/dvec4 input attribute */
@@ -462,6 +529,7 @@ nir_shader_gather_info(nir_shader *shader, nir_function_impl *entrypoint)
    shader->info.num_textures = 0;
    shader->info.num_images = 0;
    shader->info.last_msaa_image = -1;
+
    nir_foreach_variable(var, &shader->uniforms) {
       /* Bindless textures and images don't use non-bindless slots. */
       if (var->data.bindless)
@@ -483,6 +551,11 @@ nir_shader_gather_info(nir_shader *shader, nir_function_impl *entrypoint)
    shader->info.patch_inputs_read = 0;
    shader->info.patch_outputs_written = 0;
    shader->info.system_values_read = 0;
+   shader->info.inputs_read_indirectly = 0;
+   shader->info.outputs_accessed_indirectly = 0;
+   shader->info.patch_inputs_read_indirectly = 0;
+   shader->info.patch_outputs_accessed_indirectly = 0;
+
    if (shader->info.stage == MESA_SHADER_VERTEX) {
       shader->info.vs.double_inputs = 0;
    }
@@ -492,6 +565,11 @@ nir_shader_gather_info(nir_shader *shader, nir_function_impl *entrypoint)
       shader->info.fs.uses_demote = false;
       shader->info.fs.needs_helper_invocations = false;
    }
+   if (shader->info.stage == MESA_SHADER_TESS_CTRL) {
+      shader->info.tess.tcs_cross_invocation_inputs_read = 0;
+      shader->info.tess.tcs_cross_invocation_outputs_read = 0;
+   }
+
    shader->info.writes_memory = shader->info.has_transform_feedback_varyings;
 
    void *dead_ctx = ralloc_context(NULL);
diff --git a/src/compiler/shader_info.h b/src/compiler/shader_info.h
index fcd235112a0..48a32f6f4ff 100644
--- a/src/compiler/shader_info.h
+++ b/src/compiler/shader_info.h
@@ -134,6 +134,15 @@ typedef struct shader_info {
    /* Which patch outputs are read */
    uint32_t patch_outputs_read;
 
+   /* Which inputs are read indirectly (subset of inputs_read) */
+   uint64_t inputs_read_indirectly;
+   /* Which outputs are read or written indirectly */
+   uint64_t outputs_accessed_indirectly;
+   /* Which patch inputs are read indirectly (subset of patch_inputs_read) */
+   uint64_t patch_inputs_read_indirectly;
+   /* Which patch outputs are read or written indirectly */
+   uint64_t patch_outputs_accessed_indirectly;
+
    /** Bitfield of which textures are used */
    uint32_t textures_used;
 
@@ -321,6 +330,16 @@ typedef struct shader_info {
          uint8_t tcs_vertices_out;
          enum gl_tess_spacing spacing:2;
 
+         /* Bit mask of TCS per-vertex inputs (VS outputs) that are used
+          * with a vertex index that is NOT the invocation id
+          */
+         uint64_t tcs_cross_invocation_inputs_read;
+
+         /* Bit mask of TCS per-vertex outputs that are used
+          * with a vertex index that is NOT the invocation id
+          */
+         uint64_t tcs_cross_invocation_outputs_read;
+
          /** Is the vertex order counterclockwise? */
          bool ccw:1;
          bool point_mode:1;