freedreno: Allow UBWC on textures with multiple mipmap levels.

This is a backport of Jonathan Marek's UBWC work on turnip to GL.

Performance highlights from our trace set (320 frames sampled)
traces/glmark2/texture-texture-filter=mipmap.rdc:    +9.1% +/-   2.2%
traces/android/trex.rdc:                             +8.7% +/-   0.4%
traces/glmark2/desktop-effect=shadow:windows=4.rdc:  +4.2% +/-   2.5%

Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/merge_requests/3059>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/merge_requests/3059>

4 files changed, 22 insertions, 20 deletions

diff --git a/src/freedreno/fdl/fd6_layout.c b/src/freedreno/fdl/fd6_layout.c
index 30f334968ee..661d7574e8b 100644
--- a/src/freedreno/fdl/fd6_layout.c
+++ b/src/freedreno/fdl/fd6_layout.c
@@ -69,7 +69,7 @@ void
 fdl6_layout(struct fdl_layout *layout,
 		enum pipe_format format, uint32_t nr_samples,
 		uint32_t width0, uint32_t height0, uint32_t depth0,
-		uint32_t mip_levels, uint32_t array_size, bool is_3d, bool ubwc)
+		uint32_t mip_levels, uint32_t array_size, bool is_3d)
 {
 	assert(nr_samples > 0);
 	layout->width0 = width0;
@@ -81,6 +81,11 @@ fdl6_layout(struct fdl_layout *layout,
 	layout->format = format;
 	layout->nr_samples = nr_samples;
 
+	if (depth0 > 1)
+		layout->ubwc = false;
+	if (tile_alignment[layout->cpp].ubwc_blockwidth == 0)
+		layout->ubwc = false;
+
 	const struct util_format_description *format_desc =
 		util_format_description(format);
 	uint32_t depth = depth0;
@@ -115,8 +120,7 @@ fdl6_layout(struct fdl_layout *layout,
 	for (uint32_t level = 0; level < mip_levels; level++) {
 		struct fdl_slice *slice = &layout->slices[level];
 		struct fdl_slice *ubwc_slice = &layout->ubwc_slices[level];
-		uint32_t tile_mode = (ubwc ?
-				layout->tile_mode : fdl_tile_mode(layout, level));
+		uint32_t tile_mode = fdl_tile_mode(layout, level);
 		uint32_t width, height;
 
 		/* tiled levels of 3D textures are rounded up to PoT dimensions: */
@@ -176,7 +180,7 @@ fdl6_layout(struct fdl_layout *layout,
 
 		layout->size += slice->size0 * depth * layers_in_level;
 
-		if (ubwc) {
+		if (layout->ubwc) {
 			/* with UBWC every level is aligned to 4K */
 			layout->size = align(layout->size, 4096);
 
@@ -219,7 +223,7 @@ fdl6_layout(struct fdl_layout *layout,
 	 * get to program the UBWC and non-UBWC offset/strides
 	 * independently.
 	 */
-	if (ubwc) {
+	if (layout->ubwc) {
 		for (uint32_t level = 0; level < mip_levels; level++)
 			layout->slices[level].offset += layout->ubwc_layer_size * array_size;
 		layout->size += layout->ubwc_layer_size * array_size;
diff --git a/src/freedreno/fdl/freedreno_layout.c b/src/freedreno/fdl/freedreno_layout.c
index 5336683ba18..57f6388b69a 100644
--- a/src/freedreno/fdl/freedreno_layout.c
+++ b/src/freedreno/fdl/freedreno_layout.c
@@ -47,8 +47,6 @@ fdl_dump_layout(struct fdl_layout *layout)
 	for (uint32_t level = 0; level < layout->slices[level].size0; level++) {
 		struct fdl_slice *slice = &layout->slices[level];
 		struct fdl_slice *ubwc_slice = &layout->ubwc_slices[level];
-		uint32_t tile_mode = (layout->ubwc_layer_size ?
-				layout->tile_mode : fdl_tile_mode(layout, level));
 
 		fprintf(stderr, "%s: %ux%ux%u@%ux%u:\t%2u: stride=%4u, size=%6u,%6u, aligned_height=%3u, offset=0x%x,0x%x, layersz %5u,%5u tiling=%d\n",
 				util_format_name(layout->format),
@@ -62,6 +60,6 @@ fdl_dump_layout(struct fdl_layout *layout)
 				slice->size0 / (slice->pitch * layout->cpp),
 				slice->offset, ubwc_slice->offset,
 				layout->layer_size, layout->ubwc_layer_size,
-				tile_mode);
+				fdl_tile_mode(layout, level));
 	}
 }
diff --git a/src/freedreno/fdl/freedreno_layout.h b/src/freedreno/fdl/freedreno_layout.h
index 7230d337a48..f20a3056c2c 100644
--- a/src/freedreno/fdl/freedreno_layout.h
+++ b/src/freedreno/fdl/freedreno_layout.h
@@ -93,6 +93,7 @@ struct fdl_layout {
 	struct fdl_slice ubwc_slices[FDL_MAX_MIP_LEVELS];
 	uint32_t layer_size;
 	uint32_t ubwc_layer_size; /* in bytes */
+	bool ubwc : 1;
 	bool layer_first : 1;    /* see above description */
 
 	/* Note that for tiled textures, beyond a certain mipmap level (ie.
@@ -134,22 +135,20 @@ fdl_surface_offset(const struct fdl_layout *layout, unsigned level, unsigned lay
 static inline uint32_t
 fdl_ubwc_offset(const struct fdl_layout *layout, unsigned level, unsigned layer)
 {
-	/* for now this doesn't do anything clever, but when UBWC is enabled
-	 * for multi layer/level images, it will.
-	 */
-	if (layout->ubwc_layer_size) {
-		assert(level == 0);
-		assert(layer == 0);
-	}
-	return layout->ubwc_slices[0].offset;
+	const struct fdl_slice *slice = &layout->ubwc_slices[level];
+	return slice->offset + layer * layout->ubwc_layer_size;
 }
 
 static inline bool
 fdl_level_linear(const struct fdl_layout *layout, int level)
 {
+	if (layout->ubwc)
+		return false;
+
 	unsigned w = u_minify(layout->width0, level);
 	if (w < 16)
 		return true;
+
 	return false;
 }
 
@@ -165,7 +164,7 @@ fdl_tile_mode(const struct fdl_layout *layout, int level)
 static inline bool
 fdl_ubwc_enabled(const struct fdl_layout *layout, int level)
 {
-	return layout->ubwc_layer_size && fdl_tile_mode(layout, level);
+	return layout->ubwc;
 }
 
 void
@@ -175,7 +174,7 @@ void
 fdl6_layout(struct fdl_layout *layout,
 		enum pipe_format format, uint32_t nr_samples,
 		uint32_t width0, uint32_t height0, uint32_t depth0,
-		uint32_t mip_levels, uint32_t array_size, bool is_3d, bool ubwc);
+		uint32_t mip_levels, uint32_t array_size, bool is_3d);
 
 void
 fdl_dump_layout(struct fdl_layout *layout);
diff --git a/src/freedreno/vulkan/tu_image.c b/src/freedreno/vulkan/tu_image.c
index fee74a294c5..4b804ba5c24 100644
--- a/src/freedreno/vulkan/tu_image.c
+++ b/src/freedreno/vulkan/tu_image.c
@@ -150,6 +150,8 @@ tu_image_create(VkDevice _device,
    /* expect UBWC enabled if we asked for it */
    assert(modifier != DRM_FORMAT_MOD_QCOM_COMPRESSED || ubwc_enabled);
 
+   image->layout.ubwc = ubwc_enabled;
+
    fdl6_layout(&image->layout, vk_format_to_pipe_format(image->vk_format),
                image->samples,
                pCreateInfo->extent.width,
@@ -157,8 +159,7 @@ tu_image_create(VkDevice _device,
                pCreateInfo->extent.depth,
                pCreateInfo->mipLevels,
                pCreateInfo->arrayLayers,
-               pCreateInfo->imageType == VK_IMAGE_TYPE_3D,
-               ubwc_enabled);
+               pCreateInfo->imageType == VK_IMAGE_TYPE_3D);
 
    *pImage = tu_image_to_handle(image);