nir: Allow qualifiers on copy_deref and image instructions

In the next commit, we'll properly handle access qualifiers on struct
members by propagating them to load/store instructions, but these
instructions had no way to specify the qualifier.

Reviewed-by: Timothy Arceri <[email protected]>

6 files changed, 48 insertions, 12 deletions

diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h
index 6b2ebcab6fb..3977e75dc4b 100644
--- a/src/compiler/nir/nir.h
+++ b/src/compiler/nir/nir.h
@@ -1312,6 +1312,10 @@ typedef enum {
     */
    NIR_INTRINSIC_SWIZZLE_MASK,
 
+   /* Separate source/dest access flags for copies */
+   NIR_INTRINSIC_SRC_ACCESS = 21,
+   NIR_INTRINSIC_DST_ACCESS = 22,
+
    NIR_INTRINSIC_NUM_INDEX_FLAGS,
 
 } nir_intrinsic_index_flag;
@@ -1412,6 +1416,8 @@ INTRINSIC_IDX_ACCESSORS(param_idx, PARAM_IDX, unsigned)
 INTRINSIC_IDX_ACCESSORS(image_dim, IMAGE_DIM, enum glsl_sampler_dim)
 INTRINSIC_IDX_ACCESSORS(image_array, IMAGE_ARRAY, bool)
 INTRINSIC_IDX_ACCESSORS(access, ACCESS, enum gl_access_qualifier)
+INTRINSIC_IDX_ACCESSORS(src_access, SRC_ACCESS, enum gl_access_qualifier)
+INTRINSIC_IDX_ACCESSORS(dst_access, DST_ACCESS, enum gl_access_qualifier)
 INTRINSIC_IDX_ACCESSORS(format, FORMAT, unsigned)
 INTRINSIC_IDX_ACCESSORS(align_mul, ALIGN_MUL, unsigned)
 INTRINSIC_IDX_ACCESSORS(align_offset, ALIGN_OFFSET, unsigned)
diff --git a/src/compiler/nir/nir_builder.h b/src/compiler/nir/nir_builder.h
index 8ca776f733e..cef84a8914e 100644
--- a/src/compiler/nir/nir_builder.h
+++ b/src/compiler/nir/nir_builder.h
@@ -1120,15 +1120,28 @@ nir_store_deref(nir_builder *build, nir_deref_instr *deref,
 }
 
 static inline void
-nir_copy_deref(nir_builder *build, nir_deref_instr *dest, nir_deref_instr *src)
+nir_copy_deref_with_access(nir_builder *build, nir_deref_instr *dest,
+                           nir_deref_instr *src,
+                           enum gl_access_qualifier dest_access,
+                           enum gl_access_qualifier src_access)
 {
    nir_intrinsic_instr *copy =
       nir_intrinsic_instr_create(build->shader, nir_intrinsic_copy_deref);
    copy->src[0] = nir_src_for_ssa(&dest->dest.ssa);
    copy->src[1] = nir_src_for_ssa(&src->dest.ssa);
+   nir_intrinsic_set_dst_access(copy, dest_access);
+   nir_intrinsic_set_src_access(copy, src_access);
    nir_builder_instr_insert(build, &copy->instr);
 }
 
+static inline void
+nir_copy_deref(nir_builder *build, nir_deref_instr *dest, nir_deref_instr *src)
+{
+   nir_copy_deref_with_access(build, dest, src,
+                              (enum gl_access_qualifier) 0,
+                              (enum gl_access_qualifier) 0);
+}
+
 static inline nir_ssa_def *
 nir_load_var(nir_builder *build, nir_variable *var)
 {
diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py
index 155a548d1b8..c9d0a236c5a 100644
--- a/src/compiler/nir/nir_intrinsics.py
+++ b/src/compiler/nir/nir_intrinsics.py
@@ -111,6 +111,8 @@ IMAGE_DIM = "NIR_INTRINSIC_IMAGE_DIM"
 IMAGE_ARRAY = "NIR_INTRINSIC_IMAGE_ARRAY"
 # Access qualifiers for image and memory access intrinsics
 ACCESS = "NIR_INTRINSIC_ACCESS"
+DST_ACCESS = "NIR_INTRINSIC_DST_ACCESS"
+SRC_ACCESS = "NIR_INTRINSIC_SRC_ACCESS"
 # Image format for image intrinsics
 FORMAT = "NIR_INTRINSIC_FORMAT"
 # Offset or address alignment
@@ -156,7 +158,7 @@ intrinsic("load_param", dest_comp=0, indices=[PARAM_IDX], flags=[CAN_ELIMINATE])
 intrinsic("load_deref", dest_comp=0, src_comp=[-1],
           indices=[ACCESS], flags=[CAN_ELIMINATE])
 intrinsic("store_deref", src_comp=[-1, 0], indices=[WRMASK, ACCESS])
-intrinsic("copy_deref", src_comp=[-1, -1])
+intrinsic("copy_deref", src_comp=[-1, -1], indices=[DST_ACCESS, SRC_ACCESS])
 
 # Interpolation of input.  The interp_deref_at* intrinsics are similar to the
 # load_var intrinsic acting on a shader input except that they interpolate the
@@ -345,7 +347,8 @@ atomic3("atomic_counter_comp_swap")
 # either one or two additional scalar arguments with the same meaning as in
 # the ARB_shader_image_load_store specification.
 def image(name, src_comp=[], **kwargs):
-    intrinsic("image_deref_" + name, src_comp=[1] + src_comp, **kwargs)
+    intrinsic("image_deref_" + name, src_comp=[1] + src_comp,
+              indices=[ACCESS], **kwargs)
     intrinsic("image_" + name, src_comp=[1] + src_comp,
               indices=[IMAGE_DIM, IMAGE_ARRAY, FORMAT, ACCESS], **kwargs)
     intrinsic("bindless_image_" + name, src_comp=[1] + src_comp,
diff --git a/src/compiler/nir/nir_lower_var_copies.c b/src/compiler/nir/nir_lower_var_copies.c
index 0ba398698f0..e6ade733eba 100644
--- a/src/compiler/nir/nir_lower_var_copies.c
+++ b/src/compiler/nir/nir_lower_var_copies.c
@@ -56,7 +56,9 @@ emit_deref_copy_load_store(nir_builder *b,
                            nir_deref_instr *dst_deref,
                            nir_deref_instr **dst_deref_arr,
                            nir_deref_instr *src_deref,
-                           nir_deref_instr **src_deref_arr)
+                           nir_deref_instr **src_deref_arr,
+                           enum gl_access_qualifier dst_access,
+                           enum gl_access_qualifier src_access)
 {
    if (dst_deref_arr || src_deref_arr) {
       assert(dst_deref_arr && src_deref_arr);
@@ -79,14 +81,16 @@ emit_deref_copy_load_store(nir_builder *b,
                                     nir_build_deref_array_imm(b, dst_deref, i),
                                     dst_deref_arr + 1,
                                     nir_build_deref_array_imm(b, src_deref, i),
-                                    src_deref_arr + 1);
+                                    src_deref_arr + 1, dst_access, src_access);
       }
    } else {
       assert(glsl_get_bare_type(dst_deref->type) ==
              glsl_get_bare_type(src_deref->type));
       assert(glsl_type_is_vector_or_scalar(dst_deref->type));
 
-      nir_store_deref(b, dst_deref, nir_load_deref(b, src_deref), ~0);
+      nir_store_deref_with_access(b, dst_deref,
+                                  nir_load_deref_with_access(b, src_deref, src_access),
+                                  ~0, src_access);
    }
 }
 
@@ -106,7 +110,9 @@ nir_lower_deref_copy_instr(nir_builder *b, nir_intrinsic_instr *copy)
 
    b->cursor = nir_before_instr(&copy->instr);
    emit_deref_copy_load_store(b, dst_path.path[0], &dst_path.path[1],
-                                 src_path.path[0], &src_path.path[1]);
+                                 src_path.path[0], &src_path.path[1],
+                                 nir_intrinsic_dst_access(copy),
+                                 nir_intrinsic_src_access(copy));
 
    nir_deref_path_finish(&dst_path);
    nir_deref_path_finish(&src_path);
diff --git a/src/compiler/nir/nir_print.c b/src/compiler/nir/nir_print.c
index d67c6c79c17..6dac4eb5f5c 100644
--- a/src/compiler/nir/nir_print.c
+++ b/src/compiler/nir/nir_print.c
@@ -771,6 +771,8 @@ print_intrinsic_instr(nir_intrinsic_instr *instr, print_state *state)
       [NIR_INTRINSIC_IMAGE_DIM] = "image_dim",
       [NIR_INTRINSIC_IMAGE_ARRAY] = "image_array",
       [NIR_INTRINSIC_ACCESS] = "access",
+      [NIR_INTRINSIC_SRC_ACCESS] = "src-access",
+      [NIR_INTRINSIC_DST_ACCESS] = "dst-access",
       [NIR_INTRINSIC_FORMAT] = "format",
       [NIR_INTRINSIC_ALIGN_MUL] = "align_mul",
       [NIR_INTRINSIC_ALIGN_OFFSET] = "align_offset",
diff --git a/src/compiler/nir/nir_split_var_copies.c b/src/compiler/nir/nir_split_var_copies.c
index 355a4e56d01..10b71c16c17 100644
--- a/src/compiler/nir/nir_split_var_copies.c
+++ b/src/compiler/nir/nir_split_var_copies.c
@@ -64,21 +64,25 @@
 
 static void
 split_deref_copy_instr(nir_builder *b,
-                       nir_deref_instr *dst, nir_deref_instr *src)
+                       nir_deref_instr *dst, nir_deref_instr *src,
+                       enum gl_access_qualifier dst_access,
+                       enum gl_access_qualifier src_access)
 {
    assert(glsl_get_bare_type(dst->type) ==
           glsl_get_bare_type(src->type));
    if (glsl_type_is_vector_or_scalar(src->type)) {
-      nir_copy_deref(b, dst, src);
+      nir_copy_deref_with_access(b, dst, src, dst_access, src_access);
    } else if (glsl_type_is_struct_or_ifc(src->type)) {
       for (unsigned i = 0; i < glsl_get_length(src->type); i++) {
          split_deref_copy_instr(b, nir_build_deref_struct(b, dst, i),
-                                   nir_build_deref_struct(b, src, i));
+                                   nir_build_deref_struct(b, src, i),
+                                   dst_access, src_access);
       }
    } else {
       assert(glsl_type_is_matrix(src->type) || glsl_type_is_array(src->type));
       split_deref_copy_instr(b, nir_build_deref_array_wildcard(b, dst),
-                                nir_build_deref_array_wildcard(b, src));
+                                nir_build_deref_array_wildcard(b, src),
+                                dst_access, src_access);
    }
 }
 
@@ -105,7 +109,9 @@ split_var_copies_impl(nir_function_impl *impl)
             nir_instr_as_deref(copy->src[0].ssa->parent_instr);
          nir_deref_instr *src =
             nir_instr_as_deref(copy->src[1].ssa->parent_instr);
-         split_deref_copy_instr(&b, dst, src);
+         split_deref_copy_instr(&b, dst, src,
+                                nir_intrinsic_dst_access(copy),
+                                nir_intrinsic_src_access(copy));
 
          progress = true;
       }