ac: replace glc,slc with cache_policy for stores

cosmetic change

Reviewed-by: Pierre-Eric Pelloux-Prayer <[email protected]>

7 files changed, 68 insertions, 86 deletions

diff --git a/src/amd/common/ac_llvm_build.c b/src/amd/common/ac_llvm_build.c
index 4c48fe1d1d7..eb143e3fa10 100644
--- a/src/amd/common/ac_llvm_build.c
+++ b/src/amd/common/ac_llvm_build.c
@@ -1123,8 +1123,7 @@ ac_build_llvm7_buffer_store_common(struct ac_llvm_context *ctx,
 				   LLVMValueRef vindex,
 				   LLVMValueRef voffset,
 				   unsigned num_channels,
-				   bool glc,
-				   bool slc,
+				   unsigned cache_policy,
 				   bool use_format)
 {
 	LLVMValueRef args[] = {
@@ -1132,8 +1131,8 @@ ac_build_llvm7_buffer_store_common(struct ac_llvm_context *ctx,
 		LLVMBuildBitCast(ctx->builder, rsrc, ctx->v4i32, ""),
 		vindex ? vindex : ctx->i32_0,
 		voffset,
-		LLVMConstInt(ctx->i1, glc, 0),
-		LLVMConstInt(ctx->i1, slc, 0)
+		LLVMConstInt(ctx->i1, !!(cache_policy & ac_glc), 0),
+		LLVMConstInt(ctx->i1, !!(cache_policy & ac_slc), 0)
 	};
 	unsigned func = CLAMP(num_channels, 1, 3) - 1;
 
@@ -1161,8 +1160,7 @@ ac_build_llvm8_buffer_store_common(struct ac_llvm_context *ctx,
 				   LLVMValueRef soffset,
 				   unsigned num_channels,
 				   LLVMTypeRef return_channel_type,
-				   bool glc,
-				   bool slc,
+				   unsigned cache_policy,
 				   bool use_format,
 				   bool structurized)
 {
@@ -1174,7 +1172,7 @@ ac_build_llvm8_buffer_store_common(struct ac_llvm_context *ctx,
 		args[idx++] = vindex ? vindex : ctx->i32_0;
 	args[idx++] = voffset ? voffset : ctx->i32_0;
 	args[idx++] = soffset ? soffset : ctx->i32_0;
-	args[idx++] = get_cache_policy(ctx, false, glc, slc);
+	args[idx++] = LLVMConstInt(ctx->i32, cache_policy, 0);
 	unsigned func = !ac_has_vec3_support(ctx->chip_class, use_format) && num_channels == 3 ? 4 : num_channels;
 	const char *indexing_kind = structurized ? "struct" : "raw";
 	char name[256], type_name[8];
@@ -1201,17 +1199,16 @@ ac_build_buffer_store_format(struct ac_llvm_context *ctx,
 			     LLVMValueRef vindex,
 			     LLVMValueRef voffset,
 			     unsigned num_channels,
-			     bool glc,
-			     bool slc)
+			     unsigned cache_policy)
 {
 	if (HAVE_LLVM >= 0x800) {
 		ac_build_llvm8_buffer_store_common(ctx, rsrc, data, vindex,
 						   voffset, NULL, num_channels,
-						   ctx->f32, glc, slc,
+						   ctx->f32, cache_policy,
 						   true, true);
 	} else {
 		ac_build_llvm7_buffer_store_common(ctx, rsrc, data, vindex, voffset,
-						   num_channels, glc, slc,
+						   num_channels, cache_policy,
 						   true);
 	}
 }
@@ -1228,8 +1225,7 @@ ac_build_buffer_store_dword(struct ac_llvm_context *ctx,
 			    LLVMValueRef voffset,
 			    LLVMValueRef soffset,
 			    unsigned inst_offset,
-			    bool glc,
-			    bool slc,
+			    unsigned cache_policy,
 			    bool swizzle_enable_hint)
 {
 	/* Split 3 channel stores, because only LLVM 9+ support 3-channel
@@ -1244,11 +1240,11 @@ ac_build_buffer_store_dword(struct ac_llvm_context *ctx,
 		v01 = ac_build_gather_values(ctx, v, 2);
 
 		ac_build_buffer_store_dword(ctx, rsrc, v01, 2, voffset,
-					    soffset, inst_offset, glc, slc,
+					    soffset, inst_offset, cache_policy,
 					    swizzle_enable_hint);
 		ac_build_buffer_store_dword(ctx, rsrc, v[2], 1, voffset,
 					    soffset, inst_offset + 8,
-					    glc, slc,
+					    cache_policy,
 					    swizzle_enable_hint);
 		return;
 	}
@@ -1271,7 +1267,7 @@ ac_build_buffer_store_dword(struct ac_llvm_context *ctx,
 							   voffset, offset,
 							   num_channels,
 							   ctx->f32,
-							   glc, slc,
+							   cache_policy,
 							   false, false);
 		} else {
 			if (voffset)
@@ -1280,7 +1276,7 @@ ac_build_buffer_store_dword(struct ac_llvm_context *ctx,
 			ac_build_llvm7_buffer_store_common(ctx, rsrc,
 							   ac_to_float(ctx, vdata),
 							   ctx->i32_0, offset,
-							   num_channels, glc, slc,
+							   num_channels, cache_policy,
 							   false);
 		}
 		return;
@@ -1297,8 +1293,7 @@ ac_build_buffer_store_dword(struct ac_llvm_context *ctx,
 	LLVMValueRef immoffset = LLVMConstInt(ctx->i32, inst_offset, 0);
 
 	ac_build_raw_tbuffer_store(ctx, rsrc, vdata, voffset, soffset,
-			           immoffset, num_channels, dfmt, nfmt, glc,
-				   slc);
+			           immoffset, num_channels, dfmt, nfmt, cache_policy);
 }
 
 static LLVMValueRef
@@ -2045,8 +2040,7 @@ ac_build_llvm8_tbuffer_store(struct ac_llvm_context *ctx,
 			     unsigned num_channels,
 			     unsigned dfmt,
 			     unsigned nfmt,
-			     bool glc,
-			     bool slc,
+			     unsigned cache_policy,
 			     bool structurized)
 {
 	LLVMValueRef args[7];
@@ -2058,7 +2052,7 @@ ac_build_llvm8_tbuffer_store(struct ac_llvm_context *ctx,
 	args[idx++] = voffset ? voffset : ctx->i32_0;
 	args[idx++] = soffset ? soffset : ctx->i32_0;
 	args[idx++] = LLVMConstInt(ctx->i32, ac_get_tbuffer_format(ctx, dfmt, nfmt), 0);
-	args[idx++] = get_cache_policy(ctx, false, glc, slc);
+	args[idx++] = LLVMConstInt(ctx->i32, cache_policy, 0);
 	unsigned func = !ac_has_vec3_support(ctx->chip_class, true) && num_channels == 3 ? 4 : num_channels;
 	const char *indexing_kind = structurized ? "struct" : "raw";
 	char name[256], type_name[8];
@@ -2084,8 +2078,7 @@ ac_build_tbuffer_store(struct ac_llvm_context *ctx,
 		       unsigned num_channels,
 		       unsigned dfmt,
 		       unsigned nfmt,
-		       bool glc,
-		       bool slc,
+		       unsigned cache_policy,
 		       bool structurized) /* only matters for LLVM 8+ */
 {
 	if (HAVE_LLVM >= 0x800) {
@@ -2095,7 +2088,7 @@ ac_build_tbuffer_store(struct ac_llvm_context *ctx,
 
 		ac_build_llvm8_tbuffer_store(ctx, rsrc, vdata, vindex, voffset,
 					     soffset, num_channels, dfmt, nfmt,
-					     glc, slc, structurized);
+					     cache_policy, structurized);
 	} else {
 		LLVMValueRef params[] = {
 			vdata,
@@ -2106,8 +2099,8 @@ ac_build_tbuffer_store(struct ac_llvm_context *ctx,
 			immoffset,
 			LLVMConstInt(ctx->i32, dfmt, false),
 			LLVMConstInt(ctx->i32, nfmt, false),
-			LLVMConstInt(ctx->i1, glc, false),
-			LLVMConstInt(ctx->i1, slc, false),
+			LLVMConstInt(ctx->i1, !!(cache_policy & ac_glc), false),
+			LLVMConstInt(ctx->i1, !!(cache_policy & ac_slc), false),
 		};
 		unsigned func = CLAMP(num_channels, 1, 3) - 1;
 		const char *type_names[] = {"i32", "v2i32", "v4i32"};
@@ -2132,11 +2125,10 @@ ac_build_struct_tbuffer_store(struct ac_llvm_context *ctx,
 			      unsigned num_channels,
 			      unsigned dfmt,
 			      unsigned nfmt,
-			      bool glc,
-			      bool slc)
+			      unsigned cache_policy)
 {
 	ac_build_tbuffer_store(ctx, rsrc, vdata, vindex, voffset, soffset,
-			       immoffset, num_channels, dfmt, nfmt, glc, slc,
+			       immoffset, num_channels, dfmt, nfmt, cache_policy,
 			       true);
 }
 
@@ -2150,11 +2142,10 @@ ac_build_raw_tbuffer_store(struct ac_llvm_context *ctx,
 			   unsigned num_channels,
 			   unsigned dfmt,
 			   unsigned nfmt,
-			   bool glc,
-			   bool slc)
+			   unsigned cache_policy)
 {
 	ac_build_tbuffer_store(ctx, rsrc, vdata, NULL, voffset, soffset,
-			       immoffset, num_channels, dfmt, nfmt, glc, slc,
+			       immoffset, num_channels, dfmt, nfmt, cache_policy,
 			       false);
 }
 
@@ -2164,7 +2155,7 @@ ac_build_tbuffer_store_short(struct ac_llvm_context *ctx,
 			     LLVMValueRef vdata,
 			     LLVMValueRef voffset,
 			     LLVMValueRef soffset,
-			     bool glc)
+			     unsigned cache_policy)
 {
 	vdata = LLVMBuildBitCast(ctx->builder, vdata, ctx->i16, "");
 
@@ -2172,7 +2163,7 @@ ac_build_tbuffer_store_short(struct ac_llvm_context *ctx,
 		/* LLVM 9+ supports i8/i16 with struct/raw intrinsics. */
 		ac_build_llvm8_buffer_store_common(ctx, rsrc, vdata, NULL,
 						   voffset, soffset, 1,
-						   ctx->i16, glc, false,
+						   ctx->i16, cache_policy,
 						   false, false);
 	} else {
 		unsigned dfmt = V_008F0C_BUF_DATA_FORMAT_16;
@@ -2181,8 +2172,7 @@ ac_build_tbuffer_store_short(struct ac_llvm_context *ctx,
 		vdata = LLVMBuildZExt(ctx->builder, vdata, ctx->i32, "");
 
 		ac_build_raw_tbuffer_store(ctx, rsrc, vdata, voffset, soffset,
-					   ctx->i32_0, 1, dfmt, nfmt, glc,
-					   false);
+					   ctx->i32_0, 1, dfmt, nfmt, cache_policy);
 	}
 }
 
@@ -2192,7 +2182,7 @@ ac_build_tbuffer_store_byte(struct ac_llvm_context *ctx,
 			    LLVMValueRef vdata,
 			    LLVMValueRef voffset,
 			    LLVMValueRef soffset,
-			    bool glc)
+			    unsigned cache_policy)
 {
 	vdata = LLVMBuildBitCast(ctx->builder, vdata, ctx->i8, "");
 
@@ -2200,7 +2190,7 @@ ac_build_tbuffer_store_byte(struct ac_llvm_context *ctx,
 		/* LLVM 9+ supports i8/i16 with struct/raw intrinsics. */
 		ac_build_llvm8_buffer_store_common(ctx, rsrc, vdata, NULL,
 						   voffset, soffset, 1,
-						   ctx->i8, glc, false,
+						   ctx->i8, cache_policy,
 						   false, false);
 	} else {
 		unsigned dfmt = V_008F0C_BUF_DATA_FORMAT_8;
@@ -2209,7 +2199,7 @@ ac_build_tbuffer_store_byte(struct ac_llvm_context *ctx,
 		vdata = LLVMBuildZExt(ctx->builder, vdata, ctx->i32, "");
 
 		ac_build_raw_tbuffer_store(ctx, rsrc, vdata, voffset, soffset,
-					   ctx->i32_0, 1, dfmt, nfmt, glc, false);
+					   ctx->i32_0, 1, dfmt, nfmt, cache_policy);
 	}
 }
 /**
diff --git a/src/amd/common/ac_llvm_build.h b/src/amd/common/ac_llvm_build.h
index eba01e51616..edb950a9f72 100644
--- a/src/amd/common/ac_llvm_build.h
+++ b/src/amd/common/ac_llvm_build.h
@@ -268,8 +268,7 @@ ac_build_buffer_store_dword(struct ac_llvm_context *ctx,
 			    LLVMValueRef voffset,
 			    LLVMValueRef soffset,
 			    unsigned inst_offset,
-		            bool glc,
-		            bool slc,
+			    unsigned cache_policy,
 			    bool swizzle_enable_hint);
 
 void
@@ -279,8 +278,7 @@ ac_build_buffer_store_format(struct ac_llvm_context *ctx,
 			     LLVMValueRef vindex,
 			     LLVMValueRef voffset,
 			     unsigned num_channels,
-			     bool glc,
-			     bool slc);
+			     unsigned cache_policy);
 
 LLVMValueRef
 ac_build_buffer_load(struct ac_llvm_context *ctx,
@@ -392,7 +390,7 @@ ac_build_tbuffer_store_short(struct ac_llvm_context *ctx,
 			     LLVMValueRef vdata,
 			     LLVMValueRef voffset,
 			     LLVMValueRef soffset,
-			     bool glc);
+			     unsigned cache_policy);
 
 void
 ac_build_tbuffer_store_byte(struct ac_llvm_context *ctx,
@@ -400,7 +398,7 @@ ac_build_tbuffer_store_byte(struct ac_llvm_context *ctx,
 			    LLVMValueRef vdata,
 			    LLVMValueRef voffset,
 			    LLVMValueRef soffset,
-			    bool glc);
+			    unsigned cache_policy);
 
 void
 ac_build_struct_tbuffer_store(struct ac_llvm_context *ctx,
@@ -413,8 +411,7 @@ ac_build_struct_tbuffer_store(struct ac_llvm_context *ctx,
 			      unsigned num_channels,
 			      unsigned dfmt,
 			      unsigned nfmt,
-			      bool glc,
-			      bool slc);
+			      unsigned cache_policy);
 
 void
 ac_build_raw_tbuffer_store(struct ac_llvm_context *ctx,
@@ -426,8 +423,7 @@ ac_build_raw_tbuffer_store(struct ac_llvm_context *ctx,
 			   unsigned num_channels,
 			   unsigned dfmt,
 			   unsigned nfmt,
-			   bool glc,
-			   bool slc);
+			   unsigned cache_policy);
 
 LLVMValueRef
 ac_get_thread_id(struct ac_llvm_context *ctx);
diff --git a/src/amd/common/ac_nir_to_llvm.c b/src/amd/common/ac_nir_to_llvm.c
index e5a9389e7bf..69204e2395f 100644
--- a/src/amd/common/ac_nir_to_llvm.c
+++ b/src/amd/common/ac_nir_to_llvm.c
@@ -1597,11 +1597,11 @@ static void visit_store_ssbo(struct ac_nir_context *ctx,
 		if (num_bytes == 1) {
 			ac_build_tbuffer_store_byte(&ctx->ac, rsrc, data,
 						    offset, ctx->ac.i32_0,
-						    cache_policy & ac_glc);
+						    cache_policy);
 		} else if (num_bytes == 2) {
 			ac_build_tbuffer_store_short(&ctx->ac, rsrc, data,
 						     offset, ctx->ac.i32_0,
-						     cache_policy & ac_glc);
+						     cache_policy);
 		} else {
 			int num_channels = num_bytes / 4;
 
@@ -1626,8 +1626,7 @@ static void visit_store_ssbo(struct ac_nir_context *ctx,
 			ac_build_buffer_store_dword(&ctx->ac, rsrc, data,
 						    num_channels, offset,
 						    ctx->ac.i32_0, 0,
-						    cache_policy & ac_glc,
-						    false, false);
+						    cache_policy, false);
 		}
 	}
 }
@@ -2529,7 +2528,7 @@ static void visit_image_store(struct ac_nir_context *ctx,
 
 		ac_build_buffer_store_format(&ctx->ac, rsrc, src, vindex,
 					     ctx->ac.i32_0, src_channels,
-					     args.cache_policy & ac_glc, false);
+					     args.cache_policy);
 	} else {
 		args.opcode = ac_image_store;
 		args.data[0] = ac_to_float(&ctx->ac, get_src(ctx, instr->src[3]));
diff --git a/src/amd/vulkan/radv_nir_to_llvm.c b/src/amd/vulkan/radv_nir_to_llvm.c
index 43e169108e8..5ae4860b3b7 100644
--- a/src/amd/vulkan/radv_nir_to_llvm.c
+++ b/src/amd/vulkan/radv_nir_to_llvm.c
@@ -1585,13 +1585,13 @@ store_tcs_output(struct ac_shader_abi *abi,
 		if (!is_tess_factor && writemask != 0xF)
 			ac_build_buffer_store_dword(&ctx->ac, ctx->hs_ring_tess_offchip, value, 1,
 						    buf_addr, ctx->oc_lds,
-						    4 * (base + chan), 1, 0, false);
+						    4 * (base + chan), ac_glc, false);
 	}
 
 	if (writemask == 0xF) {
 		ac_build_buffer_store_dword(&ctx->ac, ctx->hs_ring_tess_offchip, src, 4,
 					    buf_addr, ctx->oc_lds,
-					    (base * 4), 1, 0, false);
+					    (base * 4), ac_glc, false);
 	}
 }
 
@@ -1858,7 +1858,7 @@ visit_emit_vertex(struct ac_shader_abi *abi, unsigned stream, LLVMValueRef *addr
 						    ctx->gsvs_ring[stream],
 						    out_val, 1,
 						    voffset, ctx->gs2vs_offset, 0,
-						    1, 1, true);
+						    ac_glc | ac_slc, true);
 		}
 	}
 
@@ -2765,7 +2765,7 @@ radv_emit_stream_output(struct radv_shader_context *ctx,
 	ac_build_buffer_store_dword(&ctx->ac, so_buffers[buf],
 				    vdata, num_comps, so_write_offsets[buf],
 				    ctx->ac.i32_0, offset,
-				    1, 1, false);
+				    ac_glc | ac_slc, false);
 }
 
 static void
@@ -3143,7 +3143,7 @@ handle_es_outputs_post(struct radv_shader_context *ctx,
 				                            out_val, 1,
 				                            NULL, ctx->es2gs_offset,
 				                            (4 * param_index + j) * 4,
-				                            1, 1, true);
+				                            ac_glc | ac_slc, true);
 			}
 		}
 	}
@@ -3276,7 +3276,7 @@ write_tess_factors(struct radv_shader_context *ctx)
 		ac_build_buffer_store_dword(&ctx->ac, buffer,
 					    LLVMConstInt(ctx->ac.i32, 0x80000000, false),
 					    1, ctx->ac.i32_0, tf_base,
-					    0, 1, 0, false);
+					    0, ac_glc, false);
 		tf_offset += 4;
 
 		ac_nir_build_endif(&inner_if_ctx);
@@ -3285,11 +3285,11 @@ write_tess_factors(struct radv_shader_context *ctx)
 	/* Store the tessellation factors. */
 	ac_build_buffer_store_dword(&ctx->ac, buffer, vec0,
 				    MIN2(stride, 4), byteoffset, tf_base,
-				    tf_offset, 1, 0, false);
+				    tf_offset, ac_glc, false);
 	if (vec1)
 		ac_build_buffer_store_dword(&ctx->ac, buffer, vec1,
 					    stride - 4, byteoffset, tf_base,
-					    16 + tf_offset, 1, 0, false);
+					    16 + tf_offset, ac_glc, false);
 
 	//store to offchip for TES to read - only if TES reads them
 	if (ctx->options->key.tcs.tes_reads_tess_factors) {
@@ -3306,7 +3306,7 @@ write_tess_factors(struct radv_shader_context *ctx)
 
 		ac_build_buffer_store_dword(&ctx->ac, ctx->hs_ring_tess_offchip, outer_vec,
 					    outer_comps, tf_outer_offset,
-					    ctx->oc_lds, 0, 1, 0, false);
+					    ctx->oc_lds, 0, ac_glc, false);
 		if (inner_comps) {
 			param_inner = shader_io_get_unique_index(VARYING_SLOT_TESS_LEVEL_INNER);
 			tf_inner_offset = get_tcs_tes_buffer_address(ctx, NULL,
@@ -3316,7 +3316,7 @@ write_tess_factors(struct radv_shader_context *ctx)
 				ac_build_gather_values(&ctx->ac, inner, inner_comps);
 			ac_build_buffer_store_dword(&ctx->ac, ctx->hs_ring_tess_offchip, inner_vec,
 						    inner_comps, tf_inner_offset,
-						    ctx->oc_lds, 0, 1, 0, false);
+						    ctx->oc_lds, 0, ac_glc, false);
 		}
 	}
 	ac_nir_build_endif(&if_ctx);
diff --git a/src/gallium/drivers/radeonsi/si_compute_prim_discard.c b/src/gallium/drivers/radeonsi/si_compute_prim_discard.c
index 42401506107..a7b0e7ad3a7 100644
--- a/src/gallium/drivers/radeonsi/si_compute_prim_discard.c
+++ b/src/gallium/drivers/radeonsi/si_compute_prim_discard.c
@@ -811,7 +811,7 @@ void si_build_prim_discard_compute_shader(struct si_shader_context *ctx)
 			};
 			LLVMValueRef rsrc = ac_build_gather_values(&ctx->ac, desc, 4);
 			ac_build_buffer_store_dword(&ctx->ac, rsrc, count, 1, ctx->i32_0,
-						    ctx->i32_0, 0, true, true, false);
+						    ctx->i32_0, 0, ac_glc | ac_slc, false);
 		} else {
 			LLVMBuildStore(builder, count,
 				       si_expand_32bit_pointer(ctx, vertex_count_addr));
@@ -863,8 +863,8 @@ void si_build_prim_discard_compute_shader(struct si_shader_context *ctx)
 			vdata = ac_build_expand_to_vec4(&ctx->ac, vdata, 3);
 
 		ac_build_buffer_store_format(&ctx->ac, output_indexbuf, vdata,
-					     vindex, ctx->i32_0, 3, true,
-					     INDEX_STORES_USE_SLC);
+					     vindex, ctx->i32_0, 3,
+					     ac_glc | (INDEX_STORES_USE_SLC ? ac_slc : 0));
 	}
 	lp_build_endif(&if_accepted);
 
diff --git a/src/gallium/drivers/radeonsi/si_shader.c b/src/gallium/drivers/radeonsi/si_shader.c
index b2f4d93bd9f..4cf642f4a95 100644
--- a/src/gallium/drivers/radeonsi/si_shader.c
+++ b/src/gallium/drivers/radeonsi/si_shader.c
@@ -1358,7 +1358,7 @@ static void store_output_tcs(struct lp_build_tgsi_context *bld_base,
 		if (reg->Register.WriteMask != 0xF && !is_tess_factor) {
 			ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
 						    buf_addr, base,
-						    4 * chan_index, 1, 0, false);
+						    4 * chan_index, ac_glc, false);
 		}
 
 		/* Write tess factors into VGPRs for the epilog. */
@@ -1378,7 +1378,7 @@ static void store_output_tcs(struct lp_build_tgsi_context *bld_base,
 		LLVMValueRef value = ac_build_gather_values(&ctx->ac,
 		                                            values, 4);
 		ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buf_addr,
-					    base, 0, 1, 0, false);
+					    base, 0, ac_glc, false);
 	}
 }
 
@@ -1486,7 +1486,7 @@ static void si_nir_store_output_tcs(struct ac_shader_abi *abi,
 			ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
 						    addr, base,
 						    4 * buffer_store_offset,
-                                                    1, 0, false);
+                                                    ac_glc, false);
 		}
 
 		/* Write tess factors into VGPRs for the epilog. */
@@ -1506,7 +1506,7 @@ static void si_nir_store_output_tcs(struct ac_shader_abi *abi,
 		LLVMValueRef value = ac_build_gather_values(&ctx->ac,
 		                                            values, 4);
 		ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, addr,
-					    base, 0, 1, 0, false);
+					    base, 0, ac_glc, false);
 	}
 }
 
@@ -2687,7 +2687,7 @@ void si_emit_streamout_output(struct si_shader_context *ctx,
 				    vdata, num_comps,
 				    so_write_offsets[buf_idx],
 				    ctx->i32_0,
-				    stream_out->dst_offset * 4, 1, 1, false);
+				    stream_out->dst_offset * 4, ac_glc | ac_slc, false);
 }
 
 /**
@@ -3091,7 +3091,7 @@ static void si_copy_tcs_inputs(struct lp_build_tgsi_context *bld_base)
 		LLVMValueRef value = lshs_lds_load(bld_base, ctx->ac.i32, ~0, lds_ptr);
 
 		ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buffer_addr,
-					    buffer_offset, 0, 1, 0, false);
+					    buffer_offset, 0, ac_glc, false);
 	}
 }
 
@@ -3217,7 +3217,7 @@ static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
 		ac_build_buffer_store_dword(&ctx->ac, buffer,
 					    LLVMConstInt(ctx->i32, 0x80000000, 0),
 					    1, ctx->i32_0, tf_base,
-					    offset, 1, 0, false);
+					    offset, ac_glc, false);
 		offset += 4;
 	}
 
@@ -3226,12 +3226,12 @@ static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
 	/* Store the tessellation factors. */
 	ac_build_buffer_store_dword(&ctx->ac, buffer, vec0,
 				    MIN2(stride, 4), byteoffset, tf_base,
-				    offset, 1, 0, false);
+				    offset, ac_glc, false);
 	offset += 16;
 	if (vec1)
 		ac_build_buffer_store_dword(&ctx->ac, buffer, vec1,
 					    stride - 4, byteoffset, tf_base,
-					    offset, 1, 0, false);
+					    offset, ac_glc, false);
 
 	/* Store the tess factors into the offchip buffer if TES reads them. */
 	if (shader->key.part.tcs.epilog.tes_reads_tess_factors) {
@@ -3254,7 +3254,7 @@ static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
 
 		ac_build_buffer_store_dword(&ctx->ac, buf, outer_vec,
 					    outer_comps, tf_outer_offset,
-					    base, 0, 1, 0, false);
+					    base, 0, ac_glc, false);
 		if (inner_comps) {
 			param_inner = si_shader_io_get_unique_index_patch(
 					      TGSI_SEMANTIC_TESSINNER, 0);
@@ -3265,7 +3265,7 @@ static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
 				    ac_build_gather_values(&ctx->ac, inner, inner_comps);
 			ac_build_buffer_store_dword(&ctx->ac, buf, inner_vec,
 						    inner_comps, tf_inner_offset,
-						    base, 0, 1, 0, false);
+						    base, 0, ac_glc, false);
 		}
 	}
 
@@ -3580,7 +3580,7 @@ static void si_llvm_emit_es_epilogue(struct ac_shader_abi *abi,
 						    ctx->esgs_ring,
 						    out_val, 1, NULL, soffset,
 						    (4 * param + chan) * 4,
-						    1, 1, true);
+						    ac_glc | ac_slc, true);
 		}
 	}
 
@@ -4306,7 +4306,7 @@ static void si_llvm_emit_vertex(struct ac_shader_abi *abi,
 						    ctx->gsvs_ring[stream],
 						    out_val, 1,
 						    voffset, soffset, 0,
-						    1, 1, true);
+						    ac_glc | ac_slc, true);
 		}
 	}
 
diff --git a/src/gallium/drivers/radeonsi/si_shader_tgsi_mem.c b/src/gallium/drivers/radeonsi/si_shader_tgsi_mem.c
index 8dcc1a552c6..74081c64599 100644
--- a/src/gallium/drivers/radeonsi/si_shader_tgsi_mem.c
+++ b/src/gallium/drivers/radeonsi/si_shader_tgsi_mem.c
@@ -635,9 +635,7 @@ static void store_emit_buffer(struct si_shader_context *ctx,
 		}
 
 		ac_build_buffer_store_dword(&ctx->ac, resource, data, count,
-					    voff, ctx->i32_0, 0,
-					    !!(cache_policy & ac_glc),
-					    !!(cache_policy & ac_slc),
+					    voff, ctx->i32_0, 0, cache_policy,
 					    false);
 	}
 }
@@ -732,8 +730,7 @@ static void store_emit(
 					     ac_build_gather_values(&ctx->ac, chans, num_channels),
 					     vindex, ctx->i32_0 /* voffset */,
 					     num_channels,
-					     !!(args.cache_policy & ac_glc),
-					     false);
+					     args.cache_policy);
 	} else {
 		args.opcode = ac_image_store;
 		args.data[0] = ac_build_gather_values(&ctx->ac, chans, 4);