freedreno: re-order support for hw queries

Push query state down to batch, and use the resource tracking to figure
out which batch(es) need to be flushed to get the query result.

This means we actually need to allocate the prsc up front, before we
know the size.  So we have to add a special way to allocate an un-
backed resource, and then later allocate the backing storage.

Signed-off-by: Rob Clark <[email protected]>

1 files changed, 112 insertions, 143 deletions

diff --git a/src/gallium/drivers/freedreno/freedreno_query_hw.c b/src/gallium/drivers/freedreno/freedreno_query_hw.c
index 808bcefc2ad..12d40d04cda 100644
--- a/src/gallium/drivers/freedreno/freedreno_query_hw.c
+++ b/src/gallium/drivers/freedreno/freedreno_query_hw.c
@@ -61,32 +61,35 @@ static int pidx(unsigned query_type)
 }
 
 static struct fd_hw_sample *
-get_sample(struct fd_context *ctx, struct fd_ringbuffer *ring,
+get_sample(struct fd_batch *batch, struct fd_ringbuffer *ring,
 		unsigned query_type)
 {
+	struct fd_context *ctx = batch->ctx;
 	struct fd_hw_sample *samp = NULL;
 	int idx = pidx(query_type);
 
 	assume(idx >= 0);   /* query never would have been created otherwise */
 
-	if (!ctx->sample_cache[idx]) {
-		ctx->sample_cache[idx] =
-			ctx->sample_providers[idx]->get_sample(ctx, ring);
-		ctx->batch->needs_flush = true;
+	if (!batch->sample_cache[idx]) {
+		struct fd_hw_sample *new_samp =
+			ctx->sample_providers[idx]->get_sample(batch, ring);
+		fd_hw_sample_reference(ctx, &batch->sample_cache[idx], new_samp);
+		util_dynarray_append(&batch->samples, struct fd_hw_sample *, new_samp);
+		batch->needs_flush = true;
 	}
 
-	fd_hw_sample_reference(ctx, &samp, ctx->sample_cache[idx]);
+	fd_hw_sample_reference(ctx, &samp, batch->sample_cache[idx]);
 
 	return samp;
 }
 
 static void
-clear_sample_cache(struct fd_context *ctx)
+clear_sample_cache(struct fd_batch *batch)
 {
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(ctx->sample_cache); i++)
-		fd_hw_sample_reference(ctx, &ctx->sample_cache[i], NULL);
+	for (i = 0; i < ARRAY_SIZE(batch->sample_cache); i++)
+		fd_hw_sample_reference(batch->ctx, &batch->sample_cache[i], NULL);
 }
 
 static bool
@@ -97,38 +100,38 @@ is_active(struct fd_hw_query *hq, enum fd_render_stage stage)
 
 
 static void
-resume_query(struct fd_context *ctx, struct fd_hw_query *hq,
+resume_query(struct fd_batch *batch, struct fd_hw_query *hq,
 		struct fd_ringbuffer *ring)
 {
 	int idx = pidx(hq->provider->query_type);
 	assert(idx >= 0);   /* query never would have been created otherwise */
 	assert(!hq->period);
-	ctx->active_providers |= (1 << idx);
-	hq->period = util_slab_alloc(&ctx->sample_period_pool);
+	batch->active_providers |= (1 << idx);
+	hq->period = util_slab_alloc(&batch->ctx->sample_period_pool);
 	list_inithead(&hq->period->list);
-	hq->period->start = get_sample(ctx, ring, hq->base.type);
+	hq->period->start = get_sample(batch, ring, hq->base.type);
 	/* NOTE: util_slab_alloc() does not zero out the buffer: */
 	hq->period->end = NULL;
 }
 
 static void
-pause_query(struct fd_context *ctx, struct fd_hw_query *hq,
+pause_query(struct fd_batch *batch, struct fd_hw_query *hq,
 		struct fd_ringbuffer *ring)
 {
 	int idx = pidx(hq->provider->query_type);
 	assert(idx >= 0);   /* query never would have been created otherwise */
 	assert(hq->period && !hq->period->end);
-	assert(ctx->active_providers & (1 << idx));
-	hq->period->end = get_sample(ctx, ring, hq->base.type);
-	list_addtail(&hq->period->list, &hq->current_periods);
+	assert(batch->active_providers & (1 << idx));
+	hq->period->end = get_sample(batch, ring, hq->base.type);
+	list_addtail(&hq->period->list, &hq->periods);
 	hq->period = NULL;
 }
 
 static void
-destroy_periods(struct fd_context *ctx, struct list_head *list)
+destroy_periods(struct fd_context *ctx, struct fd_hw_query *hq)
 {
 	struct fd_hw_sample_period *period, *s;
-	LIST_FOR_EACH_ENTRY_SAFE(period, s, list, list) {
+	LIST_FOR_EACH_ENTRY_SAFE(period, s, &hq->periods, list) {
 		fd_hw_sample_reference(ctx, &period->start, NULL);
 		fd_hw_sample_reference(ctx, &period->end, NULL);
 		list_del(&period->list);
@@ -141,8 +144,7 @@ fd_hw_destroy_query(struct fd_context *ctx, struct fd_query *q)
 {
 	struct fd_hw_query *hq = fd_hw_query(q);
 
-	destroy_periods(ctx, &hq->periods);
-	destroy_periods(ctx, &hq->current_periods);
+	destroy_periods(ctx, hq);
 	list_del(&hq->list);
 
 	free(hq);
@@ -151,27 +153,31 @@ fd_hw_destroy_query(struct fd_context *ctx, struct fd_query *q)
 static boolean
 fd_hw_begin_query(struct fd_context *ctx, struct fd_query *q)
 {
+	struct fd_batch *batch = ctx->batch;
 	struct fd_hw_query *hq = fd_hw_query(q);
+
 	if (q->active)
 		return false;
 
 	/* begin_query() should clear previous results: */
-	destroy_periods(ctx, &hq->periods);
+	destroy_periods(ctx, hq);
 
-	if (is_active(hq, ctx->stage))
-		resume_query(ctx, hq, ctx->batch->draw);
+	if (batch && is_active(hq, batch->stage))
+		resume_query(batch, hq, batch->draw);
 
 	q->active = true;
 
 	/* add to active list: */
-	list_del(&hq->list);
+	assert(list_empty(&hq->list));
 	list_addtail(&hq->list, &ctx->active_queries);
-   return true;
+
+	return true;
 }
 
 static void
 fd_hw_end_query(struct fd_context *ctx, struct fd_query *q)
 {
+	struct fd_batch *batch = ctx->batch;
 	struct fd_hw_query *hq = fd_hw_query(q);
 	/* there are a couple special cases, which don't have
 	 * a matching ->begin_query():
@@ -181,12 +187,11 @@ fd_hw_end_query(struct fd_context *ctx, struct fd_query *q)
 	}
 	if (!q->active)
 		return;
-	if (is_active(hq, ctx->stage))
-		pause_query(ctx, hq, ctx->batch->draw);
+	if (batch && is_active(hq, batch->stage))
+		pause_query(batch, hq, batch->draw);
 	q->active = false;
-	/* move to current list: */
-	list_del(&hq->list);
-	list_addtail(&hq->list, &ctx->current_queries);
+	/* remove from active list: */
+	list_delinit(&hq->list);
 }
 
 /* helper to get ptr to specified sample: */
@@ -206,27 +211,12 @@ fd_hw_get_query_result(struct fd_context *ctx, struct fd_query *q,
 	if (q->active)
 		return false;
 
-	/* if the app tries to read back the query result before the
-	 * batch is submitted, that forces us to flush so that there
-	 * are actually results to wait for:
-	 */
-	if (!LIST_IS_EMPTY(&hq->list)) {
-		/* if app didn't actually trigger any cmdstream, then
-		 * we have nothing to do:
-		 */
-		if (!ctx->batch->needs_flush)
-			return true;
-		DBG("reading query result forces flush!");
-		fd_batch_flush(ctx->batch);
-	}
-
 	util_query_clear_result(result, q->type);
 
 	if (LIST_IS_EMPTY(&hq->periods))
 		return true;
 
 	assert(LIST_IS_EMPTY(&hq->list));
-	assert(LIST_IS_EMPTY(&hq->current_periods));
 	assert(!hq->period);
 
 	/* if !wait, then check the last sample (the one most likely to
@@ -240,6 +230,21 @@ fd_hw_get_query_result(struct fd_context *ctx, struct fd_query *q,
 
 		struct fd_resource *rsc = fd_resource(period->end->prsc);
 
+		if (pending(rsc, false)) {
+			/* piglit spec@arb_occlusion_query@occlusion_query_conform
+			 * test, and silly apps perhaps, get stuck in a loop trying
+			 * to get  query result forever with wait==false..  we don't
+			 * wait to flush unnecessarily but we also don't want to
+			 * spin forever:
+			 */
+			if (hq->no_wait_cnt++ > 5)
+				fd_batch_flush(rsc->write_batch);
+			return false;
+		}
+
+		if (!rsc->bo)
+			return false;
+
 		ret = fd_bo_cpu_prep(rsc->bo, ctx->screen->pipe,
 				DRM_FREEDRENO_PREP_READ | DRM_FREEDRENO_PREP_NOSYNC);
 		if (ret)
@@ -260,6 +265,13 @@ fd_hw_get_query_result(struct fd_context *ctx, struct fd_query *q,
 
 		struct fd_resource *rsc = fd_resource(start->prsc);
 
+		if (rsc->write_batch)
+			fd_batch_flush(rsc->write_batch);
+
+		/* some piglit tests at least do query with no draws, I guess: */
+		if (!rsc->bo)
+			continue;
+
 		fd_bo_cpu_prep(rsc->bo, ctx->screen->pipe, DRM_FREEDRENO_PREP_READ);
 
 		void *ptr = fd_bo_map(rsc->bo);
@@ -299,7 +311,6 @@ fd_hw_create_query(struct fd_context *ctx, unsigned query_type)
 	hq->provider = ctx->sample_providers[idx];
 
 	list_inithead(&hq->periods);
-	list_inithead(&hq->current_periods);
 	list_inithead(&hq->list);
 
 	q = &hq->base;
@@ -310,19 +321,38 @@ fd_hw_create_query(struct fd_context *ctx, unsigned query_type)
 }
 
 struct fd_hw_sample *
-fd_hw_sample_init(struct fd_context *ctx, uint32_t size)
+fd_hw_sample_init(struct fd_batch *batch, uint32_t size)
 {
-	struct fd_hw_sample *samp = util_slab_alloc(&ctx->sample_pool);
+	struct fd_hw_sample *samp = util_slab_alloc(&batch->ctx->sample_pool);
 	pipe_reference_init(&samp->reference, 1);
 	samp->size = size;
 	debug_assert(util_is_power_of_two(size));
-	ctx->next_sample_offset = align(ctx->next_sample_offset, size);
-	samp->offset = ctx->next_sample_offset;
+	batch->next_sample_offset = align(batch->next_sample_offset, size);
+	samp->offset = batch->next_sample_offset;
 	/* NOTE: util_slab_alloc() does not zero out the buffer: */
 	samp->prsc = NULL;
 	samp->num_tiles = 0;
 	samp->tile_stride = 0;
-	ctx->next_sample_offset += size;
+	batch->next_sample_offset += size;
+
+	if (!batch->query_buf) {
+		struct pipe_screen *pscreen = &batch->ctx->screen->base;
+		struct pipe_resource templ = {
+			.target  = PIPE_BUFFER,
+			.format  = PIPE_FORMAT_R8_UNORM,
+			.bind    = PIPE_BIND_QUERY_BUFFER,
+			.width0  = 0,    /* create initially zero size buffer */
+			.height0 = 1,
+			.depth0  = 1,
+			.array_size = 1,
+			.last_level = 0,
+			.nr_samples = 1,
+		};
+		batch->query_buf = pscreen->resource_create(pscreen, &templ);
+	}
+
+	pipe_resource_reference(&samp->prsc, batch->query_buf);
+
 	return samp;
 }
 
@@ -333,110 +363,49 @@ __fd_hw_sample_destroy(struct fd_context *ctx, struct fd_hw_sample *samp)
 	util_slab_free(&ctx->sample_pool, samp);
 }
 
-static void
-prepare_sample(struct fd_hw_sample *samp, struct pipe_resource *prsc,
-		uint32_t num_tiles, uint32_t tile_stride)
-{
-	if (samp->prsc) {
-		assert(samp->prsc == prsc);
-		assert(samp->num_tiles == num_tiles);
-		assert(samp->tile_stride == tile_stride);
-		return;
-	}
-	pipe_resource_reference(&samp->prsc, prsc);
-	samp->num_tiles = num_tiles;
-	samp->tile_stride = tile_stride;
-}
-
-static void
-prepare_query(struct fd_hw_query *hq, struct pipe_resource *prsc,
-		uint32_t num_tiles, uint32_t tile_stride)
-{
-	struct fd_hw_sample_period *period, *s;
-
-	/* prepare all the samples in the query: */
-	LIST_FOR_EACH_ENTRY_SAFE(period, s, &hq->current_periods, list) {
-		prepare_sample(period->start, prsc, num_tiles, tile_stride);
-		prepare_sample(period->end, prsc, num_tiles, tile_stride);
-
-		/* move from current_periods list to periods list: */
-		list_del(&period->list);
-		list_addtail(&period->list, &hq->periods);
-	}
-}
-
-static void
-prepare_queries(struct fd_context *ctx, struct pipe_resource *prsc,
-		uint32_t num_tiles, uint32_t tile_stride,
-		struct list_head *list, bool remove)
-{
-	struct fd_hw_query *hq, *s;
-	LIST_FOR_EACH_ENTRY_SAFE(hq, s, list, list) {
-		prepare_query(hq, prsc, num_tiles, tile_stride);
-		if (remove)
-			list_delinit(&hq->list);
-	}
-}
-
 /* called from gmem code once total storage requirements are known (ie.
  * number of samples times number of tiles)
  */
 void
-fd_hw_query_prepare(struct fd_context *ctx, uint32_t num_tiles)
+fd_hw_query_prepare(struct fd_batch *batch, uint32_t num_tiles)
 {
-	uint32_t tile_stride = ctx->next_sample_offset;
-	struct pipe_resource *prsc;
+	uint32_t tile_stride = batch->next_sample_offset;
 
-	pipe_resource_reference(&ctx->query_buf, NULL);
+	if (tile_stride > 0)
+		fd_resource_resize(batch->query_buf, tile_stride * num_tiles);
 
-	if (tile_stride > 0) {
-		struct pipe_screen *pscreen = &ctx->screen->base;
-		struct pipe_resource templ = {
-			.target  = PIPE_BUFFER,
-			.format  = PIPE_FORMAT_R8_UNORM,
-			.bind    = PIPE_BIND_QUERY_BUFFER,
-			.width0  = tile_stride * num_tiles,
-			.height0 = 1,
-			.depth0  = 1,
-			.array_size = 1,
-			.last_level = 0,
-			.nr_samples = 1,
-		};
-		prsc = pscreen->resource_create(pscreen, &templ);
-	} else {
-		prsc = NULL;
-	}
-
-	ctx->query_buf = prsc;
-	ctx->query_tile_stride = tile_stride;
+	batch->query_tile_stride = tile_stride;
 
-	prepare_queries(ctx, prsc, num_tiles, tile_stride,
-			&ctx->active_queries, false);
-	prepare_queries(ctx, prsc, num_tiles, tile_stride,
-			&ctx->current_queries, true);
+	while (batch->samples.size > 0) {
+		struct fd_hw_sample *samp =
+			util_dynarray_pop(&batch->samples, struct fd_hw_sample *);
+		samp->num_tiles = num_tiles;
+		samp->tile_stride = tile_stride;
+		fd_hw_sample_reference(batch->ctx, &samp, NULL);
+	}
 
 	/* reset things for next batch: */
-	ctx->next_sample_offset = 0;
+	batch->next_sample_offset = 0;
 }
 
 void
-fd_hw_query_prepare_tile(struct fd_context *ctx, uint32_t n,
+fd_hw_query_prepare_tile(struct fd_batch *batch, uint32_t n,
 		struct fd_ringbuffer *ring)
 {
-	uint32_t tile_stride = ctx->query_tile_stride;
+	uint32_t tile_stride = batch->query_tile_stride;
 	uint32_t offset = tile_stride * n;
 
 	/* bail if no queries: */
 	if (tile_stride == 0)
 		return;
 
-	fd_wfi(ctx, ring);
+	fd_wfi(batch->ctx, ring);
 	OUT_PKT0 (ring, HW_QUERY_BASE_REG, 1);
-	OUT_RELOCW(ring, fd_resource(ctx->query_buf)->bo, offset, 0, 0);
+	OUT_RELOCW(ring, fd_resource(batch->query_buf)->bo, offset, 0, 0);
 }
 
 void
-fd_hw_query_set_stage(struct fd_context *ctx, struct fd_ringbuffer *ring,
+fd_hw_query_set_stage(struct fd_batch *batch, struct fd_ringbuffer *ring,
 		enum fd_render_stage stage)
 {
 	/* special case: internal blits (like mipmap level generation)
@@ -445,24 +414,24 @@ fd_hw_query_set_stage(struct fd_context *ctx, struct fd_ringbuffer *ring,
 	 * don't enable queries which should be paused during internal
 	 * blits:
 	 */
-	if ((ctx->stage == FD_STAGE_BLIT) &&
+	if ((batch->stage == FD_STAGE_BLIT) &&
 			(stage != FD_STAGE_NULL))
 		return;
 
-	if (stage != ctx->stage) {
+	if (stage != batch->stage) {
 		struct fd_hw_query *hq;
-		LIST_FOR_EACH_ENTRY(hq, &ctx->active_queries, list) {
-			bool was_active = is_active(hq, ctx->stage);
+		LIST_FOR_EACH_ENTRY(hq, &batch->ctx->active_queries, list) {
+			bool was_active = is_active(hq, batch->stage);
 			bool now_active = is_active(hq, stage);
 
 			if (now_active && !was_active)
-				resume_query(ctx, hq, ring);
+				resume_query(batch, hq, ring);
 			else if (was_active && !now_active)
-				pause_query(ctx, hq, ring);
+				pause_query(batch, hq, ring);
 		}
 	}
-	clear_sample_cache(ctx);
-	ctx->stage = stage;
+	clear_sample_cache(batch);
+	batch->stage = stage;
 }
 
 /* call the provider->enable() for all the hw queries that were active
@@ -470,16 +439,17 @@ fd_hw_query_set_stage(struct fd_context *ctx, struct fd_ringbuffer *ring,
  * for the duration of the batch.
  */
 void
-fd_hw_query_enable(struct fd_context *ctx, struct fd_ringbuffer *ring)
+fd_hw_query_enable(struct fd_batch *batch, struct fd_ringbuffer *ring)
 {
+	struct fd_context *ctx = batch->ctx;
 	for (int idx = 0; idx < MAX_HW_SAMPLE_PROVIDERS; idx++) {
-		if (ctx->active_providers & (1 << idx)) {
+		if (batch->active_providers & (1 << idx)) {
 			assert(ctx->sample_providers[idx]);
 			if (ctx->sample_providers[idx]->enable)
 				ctx->sample_providers[idx]->enable(ctx, ring);
 		}
 	}
-	ctx->active_providers = 0;  /* clear it for next frame */
+	batch->active_providers = 0;  /* clear it for next frame */
 }
 
 void
@@ -505,7 +475,6 @@ fd_hw_query_init(struct pipe_context *pctx)
 	util_slab_create(&ctx->sample_period_pool, sizeof(struct fd_hw_sample_period),
 			16, UTIL_SLAB_SINGLETHREADED);
 	list_inithead(&ctx->active_queries);
-	list_inithead(&ctx->current_queries);
 }
 
 void