path: root/src/gallium/drivers/radeonsi/gfx10_query.c

/*
 * Copyright 2018 Advanced Micro Devices, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * on the rights to use, copy, modify, merge, publish, distribute, sub
 * license, and/or sell copies of the Software, and to permit persons to whom
 * the Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#include <stddef.h>

#include "si_pipe.h"
#include "si_query.h"
#include "util/u_memory.h"
#include "util/u_suballoc.h"
#include "sid.h"

/**
 * The query buffer is written to by ESGS NGG shaders with statistics about
 * generated and (streamout-)emitted primitives.
 *
 * The context maintains a ring of these query buffers, and queries simply
 * point into the ring, allowing an arbitrary number of queries to be active
 * without additional GPU cost.
 */
struct gfx10_sh_query_buffer {
	struct list_head list;
	struct si_resource *buf;
	unsigned refcount;

	/* Offset into the buffer in bytes; points at the first un-emitted entry. */
	unsigned head;
};

/* Memory layout of the query buffer. Must be kept in sync with shaders
 * (including QBO shaders) and should be aligned to cachelines.
 *
 * The somewhat awkward memory layout is for compatibility with the
 * SET_PREDICATION packet, which also means that we're setting the high bit
 * of all those values unconditionally.
 */
struct gfx10_sh_query_buffer_mem {
	struct {
		uint64_t generated_primitives_start_dummy;
		uint64_t emitted_primitives_start_dummy;
		uint64_t generated_primitives;
		uint64_t emitted_primitives;
	} stream[4];
	uint32_t fence; /* bottom-of-pipe fence: set to ~0 when draws have finished */
	uint32_t pad[31];
};

/* Shader-based queries. */
struct gfx10_sh_query {
	struct si_query b;

	struct gfx10_sh_query_buffer *first;
	struct gfx10_sh_query_buffer *last;
	unsigned first_begin;
	unsigned last_end;

	unsigned stream;
};

static void emit_shader_query(struct si_context *sctx)
{
	assert(!LIST_IS_EMPTY(&sctx->shader_query_buffers));

	struct gfx10_sh_query_buffer *qbuf = list_last_entry(&sctx->shader_query_buffers,
							     struct gfx10_sh_query_buffer, list);
	qbuf->head += sizeof(struct gfx10_sh_query_buffer_mem);
}

static void gfx10_release_query_buffers(struct si_context *sctx,
					struct gfx10_sh_query_buffer *first,
					struct gfx10_sh_query_buffer *last)
{
	while (first) {
		struct gfx10_sh_query_buffer *qbuf = first;
		if (first != last)
			first = LIST_ENTRY(struct gfx10_sh_query_buffer, qbuf->list.next, list);
		else
			first = NULL;

		qbuf->refcount--;
		if (qbuf->refcount)
			continue;

		if (qbuf->list.next == &sctx->shader_query_buffers)
			continue; /* keep the most recent buffer; it may not be full yet */
		if (qbuf->list.prev == &sctx->shader_query_buffers)
			continue; /* keep the oldest buffer for recycling */

		LIST_DEL(&qbuf->list);
		si_resource_reference(&qbuf->buf, NULL);
		FREE(qbuf);
	}
}

static bool gfx10_alloc_query_buffer(struct si_context *sctx)
{
	if (si_is_atom_dirty(sctx, &sctx->atoms.s.shader_query))
		return true;

	struct gfx10_sh_query_buffer *qbuf = NULL;

	if (!LIST_IS_EMPTY(&sctx->shader_query_buffers)) {
		qbuf = list_last_entry(&sctx->shader_query_buffers,
				       struct gfx10_sh_query_buffer, list);
		if (qbuf->head + sizeof(struct gfx10_sh_query_buffer_mem) <= qbuf->buf->b.b.width0)
			goto success;

		qbuf = list_first_entry(&sctx->shader_query_buffers,
				        struct gfx10_sh_query_buffer, list);
		if (!qbuf->refcount &&
		    !si_rings_is_buffer_referenced(sctx, qbuf->buf->buf, RADEON_USAGE_READWRITE) &&
		    sctx->ws->buffer_wait(qbuf->buf->buf, 0, RADEON_USAGE_READWRITE)) {
			/* Can immediately re-use the oldest buffer */
			LIST_DEL(&qbuf->list);
		} else {
			qbuf = NULL;
		}
	}

	if (!qbuf) {
		qbuf = CALLOC_STRUCT(gfx10_sh_query_buffer);
		if (unlikely(!qbuf))
			return false;

		struct si_screen *screen = sctx->screen;
		unsigned buf_size = MAX2(sizeof(struct gfx10_sh_query_buffer_mem),
					 screen->info.min_alloc_size);
		qbuf->buf = si_resource(
			pipe_buffer_create(&screen->b, 0, PIPE_USAGE_STAGING, buf_size));
		if (unlikely(!qbuf->buf)) {
			FREE(qbuf);
			return false;
		}
	}

	/* The buffer is currently unused by the GPU. Initialize it.
	 *
	 * We need to set the high bit of all the primitive counters for
	 * compatibility with the SET_PREDICATION packet.
	 */
	uint64_t *results = sctx->ws->buffer_map(qbuf->buf->buf, NULL,
						 PIPE_TRANSFER_WRITE |
						 PIPE_TRANSFER_UNSYNCHRONIZED);
	assert(results);

	for (unsigned i = 0, e = qbuf->buf->b.b.width0 / sizeof(struct gfx10_sh_query_buffer_mem);
	     i < e; ++i) {
		for (unsigned j = 0; j < 16; ++j)
			results[32 * i + j] = (uint64_t)1 << 63;
		results[32 * i + 16] = 0;
	}

	LIST_ADDTAIL(&qbuf->list, &sctx->shader_query_buffers);
	qbuf->head = 0;
	qbuf->refcount = sctx->num_active_shader_queries;

success:;
	struct pipe_shader_buffer sbuf;
	sbuf.buffer = &qbuf->buf->b.b;
	sbuf.buffer_offset = qbuf->head;
	sbuf.buffer_size = sizeof(struct gfx10_sh_query_buffer_mem);
	si_set_rw_shader_buffer(sctx, GFX10_GS_QUERY_BUF, &sbuf);

	si_mark_atom_dirty(sctx, &sctx->atoms.s.shader_query);
	return true;
}

static void gfx10_sh_query_destroy(struct si_context *sctx, struct si_query *rquery)
{
	struct gfx10_sh_query *query = (struct gfx10_sh_query *)rquery;
	gfx10_release_query_buffers(sctx, query->first, query->last);
	FREE(query);
}

static bool gfx10_sh_query_begin(struct si_context *sctx, struct si_query *rquery)
{
	struct gfx10_sh_query *query = (struct gfx10_sh_query *)rquery;

	gfx10_release_query_buffers(sctx, query->first, query->last);
	query->first = query->last = NULL;

	if (unlikely(!gfx10_alloc_query_buffer(sctx)))
		return false;

	query->first = list_last_entry(&sctx->shader_query_buffers,
				       struct gfx10_sh_query_buffer, list);
	query->first_begin = query->first->head;

	sctx->num_active_shader_queries++;
	query->first->refcount++;

	return true;
}

static bool gfx10_sh_query_end(struct si_context *sctx, struct si_query *rquery)
{
	struct gfx10_sh_query *query = (struct gfx10_sh_query *)rquery;

	if (unlikely(!query->first))
		return false; /* earlier out of memory error */

	query->last = list_last_entry(&sctx->shader_query_buffers,
				      struct gfx10_sh_query_buffer, list);
	query->last_end = query->last->head;

	/* Signal the fence of the previous chunk */
	if (query->last_end != 0) {
		uint64_t fence_va = query->last->buf->gpu_address;
		fence_va += query->last_end - sizeof(struct gfx10_sh_query_buffer_mem);
		fence_va += offsetof(struct gfx10_sh_query_buffer_mem, fence);
		si_cp_release_mem(sctx, sctx->gfx_cs,
				  V_028A90_BOTTOM_OF_PIPE_TS, 0,
				  EOP_DST_SEL_MEM, EOP_INT_SEL_NONE,
				  EOP_DATA_SEL_VALUE_32BIT,
				  query->last->buf, fence_va, 0xffffffff,
				  PIPE_QUERY_GPU_FINISHED);
	}

	sctx->num_active_shader_queries--;

	if (sctx->num_active_shader_queries > 0) {
		gfx10_alloc_query_buffer(sctx);
	} else {
		si_set_rw_shader_buffer(sctx, GFX10_GS_QUERY_BUF, NULL);

		/* If a query_begin is followed by a query_end without a draw
		 * in-between, we need to clear the atom to ensure that the
		 * next query_begin will re-initialize the shader buffer. */
		si_set_atom_dirty(sctx, &sctx->atoms.s.shader_query, false);
	}

	return true;
}

static void gfx10_sh_query_add_result(struct gfx10_sh_query *query,
				      struct gfx10_sh_query_buffer_mem *qmem,
				      union pipe_query_result *result)
{
	static const uint64_t mask = ((uint64_t)1 << 63) - 1;

	switch (query->b.type) {
	case PIPE_QUERY_PRIMITIVES_EMITTED:
		result->u64 += qmem->stream[query->stream].emitted_primitives & mask;
		break;
	case PIPE_QUERY_PRIMITIVES_GENERATED:
		result->u64 += qmem->stream[query->stream].generated_primitives & mask;
		break;
	case PIPE_QUERY_SO_STATISTICS:
		result->so_statistics.num_primitives_written +=
			qmem->stream[query->stream].emitted_primitives & mask;
		result->so_statistics.primitives_storage_needed +=
			qmem->stream[query->stream].generated_primitives & mask;
		break;
	case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
		result->b |= qmem->stream[query->stream].emitted_primitives !=
			     qmem->stream[query->stream].generated_primitives;
		break;
	case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
		for (unsigned stream = 0; stream < SI_MAX_STREAMS; ++stream) {
			result->b |= qmem->stream[query->stream].emitted_primitives !=
				     qmem->stream[query->stream].generated_primitives;
		}
		break;
	default:
		assert(0);
	}
}

static bool gfx10_sh_query_get_result(struct si_context *sctx, struct si_query *rquery,
				      bool wait, union pipe_query_result *result)
{
	struct gfx10_sh_query *query = (struct gfx10_sh_query *)rquery;

	util_query_clear_result(result, query->b.type);

	if (unlikely(!query->first))
		return false; /* earlier out of memory error */
	assert(query->last);

	for (struct gfx10_sh_query_buffer *qbuf = query->last;;
	     qbuf = LIST_ENTRY(struct gfx10_sh_query_buffer, qbuf->list.prev, list)) {
		unsigned usage = PIPE_TRANSFER_READ |
				 (wait ? 0 : PIPE_TRANSFER_DONTBLOCK);
		void *map;

		if (rquery->b.flushed)
			map = sctx->ws->buffer_map(qbuf->buf->buf, NULL, usage);
		else
			map = si_buffer_map_sync_with_rings(sctx, qbuf->buf, usage);

		if (!map)
			return false;

		unsigned results_begin = 0;
		unsigned results_end = qbuf->head;
		if (qbuf == query->first)
			results_begin = query->first_begin;
		if (qbuf == query->last)
			results_end = query->last_end;

		while (results_begin != results_end) {
			struct gfx10_sh_query_buffer_mem *qmem = map + results_begin;
			results_begin += sizeof(*qmem);

			gfx10_sh_query_add_result(query, qmem, result);
		}

		if (qbuf == query->first)
			break;
	}

	return true;
}

static void gfx10_sh_query_get_result_resource(struct si_context *sctx,
					       struct si_query *rquery,
					       bool wait,
					       enum pipe_query_value_type result_type,
					       int index,
					       struct pipe_resource *resource,
					       unsigned offset)
{
	struct gfx10_sh_query *query = (struct gfx10_sh_query *)rquery;
	struct si_qbo_state saved_state = {};
	struct pipe_resource *tmp_buffer = NULL;
	unsigned tmp_buffer_offset = 0;

	if (!sctx->sh_query_result_shader) {
		sctx->sh_query_result_shader = gfx10_create_sh_query_result_cs(sctx);
		if (!sctx->sh_query_result_shader)
			return;
	}

	if (query->first != query->last) {
		u_suballocator_alloc(sctx->allocator_zeroed_memory, 16, 16,
				     &tmp_buffer_offset, &tmp_buffer);
		if (!tmp_buffer)
			return;
	}

	si_save_qbo_state(sctx, &saved_state);

	/* Pre-fill the constants configuring the shader behavior. */
	struct {
		uint32_t config;
		uint32_t offset;
		uint32_t chain;
		uint32_t result_count;
	} consts;
	struct pipe_constant_buffer constant_buffer = {};

	if (index >= 0) {
		switch (query->b.type) {
		case PIPE_QUERY_PRIMITIVES_GENERATED:
			consts.offset = sizeof(uint32_t) * query->stream;
			consts.config = 0;
			break;
		case PIPE_QUERY_PRIMITIVES_EMITTED:
			consts.offset = sizeof(uint32_t) * (4 + query->stream);
			consts.config = 0;
			break;
		case PIPE_QUERY_SO_STATISTICS:
			consts.offset = sizeof(uint32_t) * (4 * index + query->stream);
			consts.config = 0;
			break;
		case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
			consts.offset = sizeof(uint32_t) * query->stream;
			consts.config = 2;
			break;
		case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
			consts.offset = 0;
			consts.config = 3;
			break;
		default: unreachable("bad query type");
		}
	} else {
		/* Check result availability. */
		consts.offset = 0;
		consts.config = 1;
	}

	if (result_type == PIPE_QUERY_TYPE_I64 || result_type == PIPE_QUERY_TYPE_U64)
		consts.config |= 8;

	constant_buffer.buffer_size = sizeof(consts);
	constant_buffer.user_buffer = &consts;

	/* Pre-fill the SSBOs and grid. */
	struct pipe_shader_buffer ssbo[3];
	struct pipe_grid_info grid = {};

	ssbo[1].buffer = tmp_buffer;
	ssbo[1].buffer_offset = tmp_buffer_offset;
	ssbo[1].buffer_size = 16;

	ssbo[2] = ssbo[1];

	sctx->b.bind_compute_state(&sctx->b, sctx->sh_query_result_shader);

	grid.block[0] = 1;
	grid.block[1] = 1;
	grid.block[2] = 1;
	grid.grid[0] = 1;
	grid.grid[1] = 1;
	grid.grid[2] = 1;

	struct gfx10_sh_query_buffer *qbuf = query->first;
	for (;;) {
		unsigned begin = qbuf == query->first ? query->first_begin : 0;
		unsigned end = qbuf == query->last ? query->last_end : qbuf->buf->b.b.width0;
		if (!end)
			continue;

		ssbo[0].buffer = &qbuf->buf->b.b;
		ssbo[0].buffer_offset = begin;
		ssbo[0].buffer_size = end - begin;

		consts.result_count = (end - begin) / sizeof(struct gfx10_sh_query_buffer_mem);
		consts.chain = 0;
		if (qbuf != query->first)
			consts.chain |= 1;
		if (qbuf != query->last)
			consts.chain |= 2;

		if (qbuf == query->last) {
			ssbo[2].buffer = resource;
			ssbo[2].buffer_offset = offset;
			ssbo[2].buffer_size = 8;
		}

		sctx->b.set_constant_buffer(&sctx->b, PIPE_SHADER_COMPUTE, 0, &constant_buffer);
		sctx->b.set_shader_buffers(&sctx->b, PIPE_SHADER_COMPUTE, 0, 3, ssbo, 0x6);

		if (wait) {
			uint64_t va;

			/* Wait for result availability. Wait only for readiness
			 * of the last entry, since the fence writes should be
			 * serialized in the CP.
			 */
			va = qbuf->buf->gpu_address;
			va += end - sizeof(struct gfx10_sh_query_buffer_mem);
			va += offsetof(struct gfx10_sh_query_buffer_mem, fence);

			si_cp_wait_mem(sctx, sctx->gfx_cs, va, 0x00000001, 0x00000001, 0);
		}

		sctx->b.launch_grid(&sctx->b, &grid);
		sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH;

		if (qbuf == query->last)
			break;
		qbuf = LIST_ENTRY(struct gfx10_sh_query_buffer, qbuf->list.next, list);
	}

	si_restore_qbo_state(sctx, &saved_state);
	pipe_resource_reference(&tmp_buffer, NULL);
}

static const struct si_query_ops gfx10_sh_query_ops = {
	.destroy = gfx10_sh_query_destroy,
	.begin = gfx10_sh_query_begin,
	.end = gfx10_sh_query_end,
	.get_result = gfx10_sh_query_get_result,
	.get_result_resource = gfx10_sh_query_get_result_resource,
};

struct pipe_query *gfx10_sh_query_create(struct si_screen *screen,
					 enum pipe_query_type query_type,
					 unsigned index)
{
	struct gfx10_sh_query *query = CALLOC_STRUCT(gfx10_sh_query);
	if (unlikely(!query))
		return NULL;

	query->b.ops = &gfx10_sh_query_ops;
	query->b.type = query_type;
	query->stream = index;

	return (struct pipe_query *)query;
}

void gfx10_init_query(struct si_context *sctx)
{
	LIST_INITHEAD(&sctx->shader_query_buffers);
	sctx->atoms.s.shader_query.emit = emit_shader_query;
}

void gfx10_destroy_query(struct si_context *sctx)
{
	while (!LIST_IS_EMPTY(&sctx->shader_query_buffers)) {
		struct gfx10_sh_query_buffer *qbuf =
			list_first_entry(&sctx->shader_query_buffers,
					 struct gfx10_sh_query_buffer, list);
		LIST_DEL(&qbuf->list);

		assert(!qbuf->refcount);
		si_resource_reference(&qbuf->buf, NULL);
		FREE(qbuf);
	}
}