r600g: improve flushed depth texture handling v2

Use r600_resource_texture::flished_depth_texture for GPU access, and
allocate it in the VRAM. For transfers we'll allocate texture in the GTT
and store it in the r600_transfer::staging.

Improves performance when flushed depth texture is frequently used by the
GPU, e.g. in Lightsmark (~30%)

Signed-off-by: Vadim Girlin <[email protected]>

1 files changed, 59 insertions, 47 deletions

diff --git a/src/gallium/drivers/r600/r600_texture.c b/src/gallium/drivers/r600/r600_texture.c
index d8c5df3159c..da6634643a3 100644
--- a/src/gallium/drivers/r600/r600_texture.c
+++ b/src/gallium/drivers/r600/r600_texture.c
@@ -295,7 +295,8 @@ static int r600_init_surface(struct radeon_surface *surface,
 	if (ptex->bind & PIPE_BIND_SCANOUT) {
 		surface->flags |= RADEON_SURF_SCANOUT;
 	}
-	if (util_format_is_depth_and_stencil(ptex->format) && !is_transfer) {
+	if ((ptex->bind & PIPE_BIND_DEPTH_STENCIL) &&
+			util_format_is_depth_and_stencil(ptex->format) && !is_transfer) {
 		surface->flags |= RADEON_SURF_ZBUFFER;
 		surface->flags |= RADEON_SURF_SBUFFER;
 	}
@@ -514,7 +515,7 @@ r600_texture_create_object(struct pipe_screen *screen,
 	rtex->real_format = base->format;
 
 	/* We must split depth and stencil into two separate buffers on Evergreen. */
-	if (!(base->flags & R600_RESOURCE_FLAG_TRANSFER) &&
+	if ((base->bind & PIPE_BIND_DEPTH_STENCIL) &&
 	    ((struct r600_screen*)screen)->chip_class >= EVERGREEN &&
 	    util_format_is_depth_and_stencil(base->format) &&
 	    !rscreen->use_surface_alloc) {
@@ -558,7 +559,8 @@ r600_texture_create_object(struct pipe_screen *screen,
 	}
 
 	/* only mark depth textures the HW can hit as depth textures */
-	if (util_format_is_depth_or_stencil(rtex->real_format) && permit_hardware_blit(screen, base))
+	if (util_format_is_depth_or_stencil(rtex->real_format) &&
+			permit_hardware_blit(screen, base))
 		rtex->is_depth = true;
 
 	r600_setup_miptree(screen, rtex, array_mode);
@@ -726,12 +728,15 @@ struct pipe_resource *r600_texture_from_handle(struct pipe_screen *screen,
 }
 
 void r600_init_flushed_depth_texture(struct pipe_context *ctx,
-				     struct pipe_resource *texture)
+				     struct pipe_resource *texture,
+				     struct r600_resource_texture **staging)
 {
 	struct r600_resource_texture *rtex = (struct r600_resource_texture*)texture;
 	struct pipe_resource resource;
+	struct r600_resource_texture **flushed_depth_texture = staging ?
+			staging : &rtex->flushed_depth_texture;
 
-	if (rtex->flushed_depth_texture)
+	if (!staging && rtex->flushed_depth_texture)
 		return; /* it's ready */
 
 	resource.target = texture->target;
@@ -742,32 +747,44 @@ void r600_init_flushed_depth_texture(struct pipe_context *ctx,
 	resource.array_size = texture->array_size;
 	resource.last_level = texture->last_level;
 	resource.nr_samples = texture->nr_samples;
-	resource.usage = PIPE_USAGE_DYNAMIC;
-	resource.bind = texture->bind | PIPE_BIND_DEPTH_STENCIL;
-	resource.flags = R600_RESOURCE_FLAG_TRANSFER | texture->flags;
+	resource.usage = staging ? PIPE_USAGE_DYNAMIC : PIPE_USAGE_DEFAULT;
+	resource.bind = texture->bind & ~PIPE_BIND_DEPTH_STENCIL;
+	resource.flags = texture->flags;
+
+	if (staging)
+		resource.flags |= R600_RESOURCE_FLAG_TRANSFER;
+	else
+		rtex->dirty_db = TRUE;
 
-	rtex->flushed_depth_texture = (struct r600_resource_texture *)ctx->screen->resource_create(ctx->screen, &resource);
-	if (rtex->flushed_depth_texture == NULL) {
-		R600_ERR("failed to create temporary texture to hold untiled copy\n");
+	*flushed_depth_texture = (struct r600_resource_texture *)ctx->screen->resource_create(ctx->screen, &resource);
+	if (*flushed_depth_texture == NULL) {
+		R600_ERR("failed to create temporary texture to hold flushed depth\n");
 		return;
 	}
 
-	((struct r600_resource_texture *)rtex->flushed_depth_texture)->is_flushing_texture = TRUE;
+	(*flushed_depth_texture)->is_flushing_texture = TRUE;
+
 }
 
 void r600_texture_depth_flush(struct pipe_context *ctx,
-			      struct pipe_resource *texture)
+			      struct pipe_resource *texture,
+			      struct r600_resource_texture **staging)
 {
 	struct r600_resource_texture *rtex = (struct r600_resource_texture*)texture;
 
-	r600_init_flushed_depth_texture(ctx, texture);
+	r600_init_flushed_depth_texture(ctx, texture, staging);
 
-	if (!rtex->flushed_depth_texture)
-		return; /* error */
+	if (staging) {
+		if (!*staging)
+			return; /* error */
 
-	/* XXX: only do this if the depth texture has actually changed:
-	 */
-	r600_blit_uncompress_depth(ctx, rtex);
+		r600_blit_uncompress_depth(ctx, rtex, *staging);
+	} else {
+		if (!rtex->flushed_depth_texture)
+			return; /* error */
+
+		r600_blit_uncompress_depth(ctx, rtex, NULL);
+	}
 }
 
 /* Needs adjustment for pixelformat:
@@ -831,15 +848,18 @@ struct pipe_transfer* r600_texture_get_transfer(struct pipe_context *ctx,
 		*/
 		/* XXX: when discard is true, no need to read back from depth texture
 		*/
-		r600_texture_depth_flush(ctx, texture);
-		if (!rtex->flushed_depth_texture) {
+		struct r600_resource_texture *staging_depth;
+
+		r600_texture_depth_flush(ctx, texture, &staging_depth);
+		if (!staging_depth) {
 			R600_ERR("failed to create temporary texture to hold untiled copy\n");
 			pipe_resource_reference(&trans->transfer.resource, NULL);
 			FREE(trans);
 			return NULL;
 		}
-		trans->transfer.stride = rtex->flushed_depth_texture->pitch_in_bytes[level];
-		trans->offset = r600_texture_get_offset(rtex->flushed_depth_texture, level, box->z);
+		trans->transfer.stride = staging_depth->pitch_in_bytes[level];
+		trans->offset = r600_texture_get_offset(staging_depth, level, box->z);
+		trans->staging = (struct r600_resource*)staging_depth;
 		return &trans->transfer;
 	} else if (use_staging_texture) {
 		resource.target = PIPE_TEXTURE_2D;
@@ -894,25 +914,25 @@ void r600_texture_transfer_destroy(struct pipe_context *ctx,
 	struct pipe_resource *texture = transfer->resource;
 	struct r600_resource_texture *rtex = (struct r600_resource_texture*)texture;
 
-	if (rtransfer->staging) {
-		if (transfer->usage & PIPE_TRANSFER_WRITE) {
-			r600_copy_from_staging_texture(ctx, rtransfer);
-		}
-		pipe_resource_reference((struct pipe_resource**)&rtransfer->staging, NULL);
-	}
-
-	if (rtex->is_depth && !rtex->is_flushing_texture) {
-		if ((transfer->usage & PIPE_TRANSFER_WRITE) && rtex->flushed_depth_texture) {
+	if (rtex->is_depth) {
+		if ((transfer->usage & PIPE_TRANSFER_WRITE) && rtransfer->staging) {
 			struct pipe_box sbox;
 
 			u_box_origin_2d(texture->width0, texture->height0, &sbox);
 
 			ctx->resource_copy_region(ctx, texture, 0, 0, 0, 0,
-						  &rtex->flushed_depth_texture->resource.b.b, 0,
+						  &rtransfer->staging->b.b, 0,
 						  &sbox);
 		}
+	} else if (rtransfer->staging) {
+		if (transfer->usage & PIPE_TRANSFER_WRITE) {
+			r600_copy_from_staging_texture(ctx, rtransfer);
+		}
 	}
 
+	if (rtransfer->staging)
+		pipe_resource_reference((struct pipe_resource**)&rtransfer->staging, NULL);
+
 	pipe_resource_reference(&transfer->resource, NULL);
 	FREE(transfer);
 }
@@ -923,6 +943,8 @@ void* r600_texture_transfer_map(struct pipe_context *ctx,
 	struct r600_context *rctx = (struct r600_context *)ctx;
 	struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
 	struct radeon_winsys_cs_handle *buf;
+	struct r600_resource_texture *rtex =
+			(struct r600_resource_texture*)transfer->resource;
 	enum pipe_format format = transfer->resource->format;
 	unsigned offset = 0;
 	char *map;
@@ -934,17 +956,13 @@ void* r600_texture_transfer_map(struct pipe_context *ctx,
 	if (rtransfer->staging) {
 		buf = ((struct r600_resource *)rtransfer->staging)->cs_buf;
 	} else {
-		struct r600_resource_texture *rtex = (struct r600_resource_texture*)transfer->resource;
-
-		if (rtex->flushed_depth_texture)
-			buf = ((struct r600_resource *)rtex->flushed_depth_texture)->cs_buf;
-		else
-			buf = ((struct r600_resource *)transfer->resource)->cs_buf;
+		buf = ((struct r600_resource *)transfer->resource)->cs_buf;
+	}
 
+	if (rtex->is_depth || !rtransfer->staging)
 		offset = rtransfer->offset +
 			transfer->box.y / util_format_get_blockheight(format) * transfer->stride +
 			transfer->box.x / util_format_get_blockwidth(format) * util_format_get_blocksize(format);
-	}
 
 	if (!(map = rctx->ws->buffer_map(buf, rctx->cs, transfer->usage))) {
 		return NULL;
@@ -967,13 +985,7 @@ void r600_texture_transfer_unmap(struct pipe_context *ctx,
 	if (rtransfer->staging) {
 		buf = ((struct r600_resource *)rtransfer->staging)->cs_buf;
 	} else {
-		struct r600_resource_texture *rtex = (struct r600_resource_texture*)transfer->resource;
-
-		if (rtex->flushed_depth_texture) {
-			buf = ((struct r600_resource *)rtex->flushed_depth_texture)->cs_buf;
-		} else {
-			buf = ((struct r600_resource *)transfer->resource)->cs_buf;
-		}
+		buf = ((struct r600_resource *)transfer->resource)->cs_buf;
 	}
 	rctx->ws->buffer_unmap(buf);
 }