/*
 * Copyright 2015 Advanced Micro Devices, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "si_build_pm4.h"
#include "si_query.h"
#include "util/u_memory.h"


enum si_pc_block_flags {
	/* This block is part of the shader engine */
	SI_PC_BLOCK_SE = (1 << 0),

	/* Expose per-instance groups instead of summing all instances (within
	 * an SE). */
	SI_PC_BLOCK_INSTANCE_GROUPS = (1 << 1),

	/* Expose per-SE groups instead of summing instances across SEs. */
	SI_PC_BLOCK_SE_GROUPS = (1 << 2),

	/* Shader block */
	SI_PC_BLOCK_SHADER = (1 << 3),

	/* Non-shader block with perfcounters windowed by shaders. */
	SI_PC_BLOCK_SHADER_WINDOWED = (1 << 4),
};

enum si_pc_reg_layout {
	/* All secondary selector dwords follow as one block after the primary
	 * selector dwords for the counters that have secondary selectors.
	 */
	SI_PC_MULTI_BLOCK = 0,

	/* Each secondary selector dword follows immediately afters the
	 * corresponding primary.
	 */
	SI_PC_MULTI_ALTERNATE = 1,

	/* All secondary selector dwords follow as one block after all primary
	 * selector dwords.
	 */
	SI_PC_MULTI_TAIL = 2,

	/* Free-form arrangement of selector registers. */
	SI_PC_MULTI_CUSTOM = 3,

	SI_PC_MULTI_MASK = 3,

	/* Registers are laid out in decreasing rather than increasing order. */
	SI_PC_REG_REVERSE = 4,

	SI_PC_FAKE = 8,
};

struct si_pc_block_base {
	const char *name;
	unsigned num_counters;
	unsigned flags;

	unsigned select_or;
	unsigned select0;
	unsigned counter0_lo;
	unsigned *select;
	unsigned *counters;
	unsigned num_multi;
	unsigned num_prelude;
	unsigned layout;
};

struct si_pc_block_gfxdescr {
	struct si_pc_block_base *b;
	unsigned selectors;
	unsigned instances;
};

struct si_pc_block {
	const struct si_pc_block_gfxdescr *b;
	unsigned num_instances;

	unsigned num_groups;
	char *group_names;
	unsigned group_name_stride;

	char *selector_names;
	unsigned selector_name_stride;
};

/* The order is chosen to be compatible with GPUPerfStudio's hardcoding of
 * performance counter group IDs.
 */
static const char * const si_pc_shader_type_suffixes[] = {
	"", "_ES", "_GS", "_VS", "_PS", "_LS", "_HS", "_CS"
};

static const unsigned si_pc_shader_type_bits[] = {
	0x7f,
	S_036780_ES_EN(1),
	S_036780_GS_EN(1),
	S_036780_VS_EN(1),
	S_036780_PS_EN(1),
	S_036780_LS_EN(1),
	S_036780_HS_EN(1),
	S_036780_CS_EN(1),
};

/* Max counters per HW block */
#define SI_QUERY_MAX_COUNTERS 16

#define SI_PC_SHADERS_WINDOWING (1 << 31)

struct si_query_group {
	struct si_query_group *next;
	struct si_pc_block *block;
	unsigned sub_gid; /* only used during init */
	unsigned result_base; /* only used during init */
	int se;
	int instance;
	unsigned num_counters;
	unsigned selectors[SI_QUERY_MAX_COUNTERS];
};

struct si_query_counter {
	unsigned base;
	unsigned qwords;
	unsigned stride; /* in uint64s */
};

struct si_query_pc {
	struct si_query b;
	struct si_query_buffer buffer;

	/* Size of the results in memory, in bytes. */
	unsigned result_size;

	unsigned shaders;
	unsigned num_counters;
	struct si_query_counter *counters;
	struct si_query_group *groups;
};


static struct si_pc_block_base cik_CB = {
	.name = "CB",
	.num_counters = 4,
	.flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS,

	.select0 = R_037000_CB_PERFCOUNTER_FILTER,
	.counter0_lo = R_035018_CB_PERFCOUNTER0_LO,
	.num_multi = 1,
	.num_prelude = 1,
	.layout = SI_PC_MULTI_ALTERNATE,
};

static unsigned cik_CPC_select[] = {
	R_036024_CPC_PERFCOUNTER0_SELECT,
	R_036010_CPC_PERFCOUNTER0_SELECT1,
	R_03600C_CPC_PERFCOUNTER1_SELECT,
};
static struct si_pc_block_base cik_CPC = {
	.name = "CPC",
	.num_counters = 2,

	.select = cik_CPC_select,
	.counter0_lo = R_034018_CPC_PERFCOUNTER0_LO,
	.num_multi = 1,
	.layout = SI_PC_MULTI_CUSTOM | SI_PC_REG_REVERSE,
};

static struct si_pc_block_base cik_CPF = {
	.name = "CPF",
	.num_counters = 2,

	.select0 = R_03601C_CPF_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034028_CPF_PERFCOUNTER0_LO,
	.num_multi = 1,
	.layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE,
};

static struct si_pc_block_base cik_CPG = {
	.name = "CPG",
	.num_counters = 2,

	.select0 = R_036008_CPG_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034008_CPG_PERFCOUNTER0_LO,
	.num_multi = 1,
	.layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE,
};

static struct si_pc_block_base cik_DB = {
	.name = "DB",
	.num_counters = 4,
	.flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS,

	.select0 = R_037100_DB_PERFCOUNTER0_SELECT,
	.counter0_lo = R_035100_DB_PERFCOUNTER0_LO,
	.num_multi = 3, // really only 2, but there's a gap between registers
	.layout = SI_PC_MULTI_ALTERNATE,
};

static struct si_pc_block_base cik_GDS = {
	.name = "GDS",
	.num_counters = 4,

	.select0 = R_036A00_GDS_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034A00_GDS_PERFCOUNTER0_LO,
	.num_multi = 1,
	.layout = SI_PC_MULTI_TAIL,
};

static unsigned cik_GRBM_counters[] = {
	R_034100_GRBM_PERFCOUNTER0_LO,
	R_03410C_GRBM_PERFCOUNTER1_LO,
};
static struct si_pc_block_base cik_GRBM = {
	.name = "GRBM",
	.num_counters = 2,

	.select0 = R_036100_GRBM_PERFCOUNTER0_SELECT,
	.counters = cik_GRBM_counters,
};

static struct si_pc_block_base cik_GRBMSE = {
	.name = "GRBMSE",
	.num_counters = 4,

	.select0 = R_036108_GRBM_SE0_PERFCOUNTER_SELECT,
	.counter0_lo = R_034114_GRBM_SE0_PERFCOUNTER_LO,
};

static struct si_pc_block_base cik_IA = {
	.name = "IA",
	.num_counters = 4,

	.select0 = R_036210_IA_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034220_IA_PERFCOUNTER0_LO,
	.num_multi = 1,
	.layout = SI_PC_MULTI_TAIL,
};

static struct si_pc_block_base cik_PA_SC = {
	.name = "PA_SC",
	.num_counters = 8,
	.flags = SI_PC_BLOCK_SE,

	.select0 = R_036500_PA_SC_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034500_PA_SC_PERFCOUNTER0_LO,
	.num_multi = 1,
	.layout = SI_PC_MULTI_ALTERNATE,
};

/* According to docs, PA_SU counters are only 48 bits wide. */
static struct si_pc_block_base cik_PA_SU = {
	.name = "PA_SU",
	.num_counters = 4,
	.flags = SI_PC_BLOCK_SE,

	.select0 = R_036400_PA_SU_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034400_PA_SU_PERFCOUNTER0_LO,
	.num_multi = 2,
	.layout = SI_PC_MULTI_ALTERNATE,
};

static struct si_pc_block_base cik_SPI = {
	.name = "SPI",
	.num_counters = 6,
	.flags = SI_PC_BLOCK_SE,

	.select0 = R_036600_SPI_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034604_SPI_PERFCOUNTER0_LO,
	.num_multi = 4,
	.layout = SI_PC_MULTI_BLOCK,
};

static struct si_pc_block_base cik_SQ = {
	.name = "SQ",
	.num_counters = 16,
	.flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_SHADER,

	.select0 = R_036700_SQ_PERFCOUNTER0_SELECT,
	.select_or = S_036700_SQC_BANK_MASK(15) |
			S_036700_SQC_CLIENT_MASK(15) |
			S_036700_SIMD_MASK(15),
	.counter0_lo = R_034700_SQ_PERFCOUNTER0_LO,
};

static struct si_pc_block_base cik_SX = {
	.name = "SX",
	.num_counters = 4,
	.flags = SI_PC_BLOCK_SE,

	.select0 = R_036900_SX_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034900_SX_PERFCOUNTER0_LO,
	.num_multi = 2,
	.layout = SI_PC_MULTI_TAIL,
};

static struct si_pc_block_base cik_TA = {
	.name = "TA",
	.num_counters = 2,
	.flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED,

	.select0 = R_036B00_TA_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034B00_TA_PERFCOUNTER0_LO,
	.num_multi = 1,
	.layout = SI_PC_MULTI_ALTERNATE,
};

static struct si_pc_block_base cik_TD = {
	.name = "TD",
	.num_counters = 2,
	.flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED,

	.select0 = R_036C00_TD_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034C00_TD_PERFCOUNTER0_LO,
	.num_multi = 1,
	.layout = SI_PC_MULTI_ALTERNATE,
};

static struct si_pc_block_base cik_TCA = {
	.name = "TCA",
	.num_counters = 4,
	.flags = SI_PC_BLOCK_INSTANCE_GROUPS,

	.select0 = R_036E40_TCA_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034E40_TCA_PERFCOUNTER0_LO,
	.num_multi = 2,
	.layout = SI_PC_MULTI_ALTERNATE,
};

static struct si_pc_block_base cik_TCC = {
	.name = "TCC",
	.num_counters = 4,
	.flags = SI_PC_BLOCK_INSTANCE_GROUPS,

	.select0 = R_036E00_TCC_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034E00_TCC_PERFCOUNTER0_LO,
	.num_multi = 2,
	.layout = SI_PC_MULTI_ALTERNATE,
};

static struct si_pc_block_base cik_TCP = {
	.name = "TCP",
	.num_counters = 4,
	.flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS | SI_PC_BLOCK_SHADER_WINDOWED,

	.select0 = R_036D00_TCP_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034D00_TCP_PERFCOUNTER0_LO,
	.num_multi = 2,
	.layout = SI_PC_MULTI_ALTERNATE,
};

static struct si_pc_block_base cik_VGT = {
	.name = "VGT",
	.num_counters = 4,
	.flags = SI_PC_BLOCK_SE,

	.select0 = R_036230_VGT_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034240_VGT_PERFCOUNTER0_LO,
	.num_multi = 1,
	.layout = SI_PC_MULTI_TAIL,
};

static struct si_pc_block_base cik_WD = {
	.name = "WD",
	.num_counters = 4,

	.select0 = R_036200_WD_PERFCOUNTER0_SELECT,
	.counter0_lo = R_034200_WD_PERFCOUNTER0_LO,
};

static struct si_pc_block_base cik_MC = {
	.name = "MC",
	.num_counters = 4,

	.layout = SI_PC_FAKE,
};

static struct si_pc_block_base cik_SRBM = {
	.name = "SRBM",
	.num_counters = 2,

	.layout = SI_PC_FAKE,
};

/* Both the number of instances and selectors varies between chips of the same
 * class. We only differentiate by class here and simply expose the maximum
 * number over all chips in a class.
 *
 * Unfortunately, GPUPerfStudio uses the order of performance counter groups
 * blindly once it believes it has identified the hardware, so the order of
 * blocks here matters.
 */
static struct si_pc_block_gfxdescr groups_CIK[] = {
	{ &cik_CB, 226},
	{ &cik_CPF, 17 },
	{ &cik_DB, 257},
	{ &cik_GRBM, 34 },
	{ &cik_GRBMSE, 15 },
	{ &cik_PA_SU, 153 },
	{ &cik_PA_SC, 395 },
	{ &cik_SPI, 186 },
	{ &cik_SQ, 252 },
	{ &cik_SX, 32 },
	{ &cik_TA, 111, 11 },
	{ &cik_TCA, 39, 2 },
	{ &cik_TCC, 160},
	{ &cik_TD, 55, 11 },
	{ &cik_TCP, 154, 11 },
	{ &cik_GDS, 121 },
	{ &cik_VGT, 140 },
	{ &cik_IA, 22 },
	{ &cik_MC, 22 },
	{ &cik_SRBM, 19 },
	{ &cik_WD, 22 },
	{ &cik_CPG, 46 },
	{ &cik_CPC, 22 },

};

static struct si_pc_block_gfxdescr groups_VI[] = {
	{ &cik_CB, 405},
	{ &cik_CPF, 19 },
	{ &cik_DB, 257},
	{ &cik_GRBM, 34 },
	{ &cik_GRBMSE, 15 },
	{ &cik_PA_SU, 154 },
	{ &cik_PA_SC, 397 },
	{ &cik_SPI, 197 },
	{ &cik_SQ, 273 },
	{ &cik_SX, 34 },
	{ &cik_TA, 119, 16 },
	{ &cik_TCA, 35, 2 },
	{ &cik_TCC, 192},
	{ &cik_TD, 55, 16 },
	{ &cik_TCP, 180, 16 },
	{ &cik_GDS, 121 },
	{ &cik_VGT, 147 },
	{ &cik_IA, 24 },
	{ &cik_MC, 22 },
	{ &cik_SRBM, 27 },
	{ &cik_WD, 37 },
	{ &cik_CPG, 48 },
	{ &cik_CPC, 24 },

};

static struct si_pc_block_gfxdescr groups_gfx9[] = {
	{ &cik_CB, 438},
	{ &cik_CPF, 32 },
	{ &cik_DB, 328},
	{ &cik_GRBM, 38 },
	{ &cik_GRBMSE, 16 },
	{ &cik_PA_SU, 292 },
	{ &cik_PA_SC, 491 },
	{ &cik_SPI, 196 },
	{ &cik_SQ, 374 },
	{ &cik_SX, 208 },
	{ &cik_TA, 119, 16 },
	{ &cik_TCA, 35, 2 },
	{ &cik_TCC, 256},
	{ &cik_TD, 57, 16 },
	{ &cik_TCP, 85, 16 },
	{ &cik_GDS, 121 },
	{ &cik_VGT, 148 },
	{ &cik_IA, 32 },
	{ &cik_WD, 58 },
	{ &cik_CPG, 59 },
	{ &cik_CPC, 35 },
};

static bool si_pc_block_has_per_se_groups(const struct si_perfcounters *pc,
					  const struct si_pc_block *block)
{
	return block->b->b->flags & SI_PC_BLOCK_SE_GROUPS ||
	       (block->b->b->flags & SI_PC_BLOCK_SE && pc->separate_se);
}

static bool si_pc_block_has_per_instance_groups(const struct si_perfcounters *pc,
						const struct si_pc_block *block)
{
	return block->b->b->flags & SI_PC_BLOCK_INSTANCE_GROUPS ||
	       (block->num_instances > 1 && pc->separate_instance);
}

static struct si_pc_block *
lookup_counter(struct si_perfcounters *pc, unsigned index,
	       unsigned *base_gid, unsigned *sub_index)
{
	struct si_pc_block *block = pc->blocks;
	unsigned bid;

	*base_gid = 0;
	for (bid = 0; bid < pc->num_blocks; ++bid, ++block) {
		unsigned total = block->num_groups * block->b->selectors;

		if (index < total) {
			*sub_index = index;
			return block;
		}

		index -= total;
		*base_gid += block->num_groups;
	}

	return NULL;
}

static struct si_pc_block *
lookup_group(struct si_perfcounters *pc, unsigned *index)
{
	unsigned bid;
	struct si_pc_block *block = pc->blocks;

	for (bid = 0; bid < pc->num_blocks; ++bid, ++block) {
		if (*index < block->num_groups)
			return block;
		*index -= block->num_groups;
	}

	return NULL;
}

static void si_pc_emit_instance(struct si_context *sctx,
				int se, int instance)
{
	struct radeon_cmdbuf *cs = sctx->gfx_cs;
	unsigned value = S_030800_SH_BROADCAST_WRITES(1);

	if (se >= 0) {
		value |= S_030800_SE_INDEX(se);
	} else {
		value |= S_030800_SE_BROADCAST_WRITES(1);
	}

	if (instance >= 0) {
		value |= S_030800_INSTANCE_INDEX(instance);
	} else {
		value |= S_030800_INSTANCE_BROADCAST_WRITES(1);
	}

	radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, value);
}

static void si_pc_emit_shaders(struct si_context *sctx,
			       unsigned shaders)
{
	struct radeon_cmdbuf *cs = sctx->gfx_cs;

	radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2);
	radeon_emit(cs, shaders & 0x7f);
	radeon_emit(cs, 0xffffffff);
}

static void si_pc_emit_select(struct si_context *sctx,
		        struct si_pc_block *block,
		        unsigned count, unsigned *selectors)
{
	struct si_pc_block_base *regs = block->b->b;
	struct radeon_cmdbuf *cs = sctx->gfx_cs;
	unsigned idx;
	unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK;
	unsigned dw;

	assert(count <= regs->num_counters);

	if (regs->layout & SI_PC_FAKE)
		return;

	if (layout_multi == SI_PC_MULTI_BLOCK) {
		assert(!(regs->layout & SI_PC_REG_REVERSE));

		dw = count + regs->num_prelude;
		if (count >= regs->num_multi)
			dw += regs->num_multi;
		radeon_set_uconfig_reg_seq(cs, regs->select0, dw);
		for (idx = 0; idx < regs->num_prelude; ++idx)
			radeon_emit(cs, 0);
		for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx)
			radeon_emit(cs, selectors[idx] | regs->select_or);

		if (count < regs->num_multi) {
			unsigned select1 =
				regs->select0 + 4 * regs->num_multi;
			radeon_set_uconfig_reg_seq(cs, select1, count);
		}

		for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx)
			radeon_emit(cs, 0);

		if (count > regs->num_multi) {
			for (idx = regs->num_multi; idx < count; ++idx)
				radeon_emit(cs, selectors[idx] | regs->select_or);
		}
	} else if (layout_multi == SI_PC_MULTI_TAIL) {
		unsigned select1, select1_count;

		assert(!(regs->layout & SI_PC_REG_REVERSE));

		radeon_set_uconfig_reg_seq(cs, regs->select0, count + regs->num_prelude);
		for (idx = 0; idx < regs->num_prelude; ++idx)
			radeon_emit(cs, 0);
		for (idx = 0; idx < count; ++idx)
			radeon_emit(cs, selectors[idx] | regs->select_or);

		select1 = regs->select0 + 4 * regs->num_counters;
		select1_count = MIN2(count, regs->num_multi);
		radeon_set_uconfig_reg_seq(cs, select1, select1_count);
		for (idx = 0; idx < select1_count; ++idx)
			radeon_emit(cs, 0);
	} else if (layout_multi == SI_PC_MULTI_CUSTOM) {
		unsigned *reg = regs->select;
		for (idx = 0; idx < count; ++idx) {
			radeon_set_uconfig_reg(cs, *reg++, selectors[idx] | regs->select_or);
			if (idx < regs->num_multi)
				radeon_set_uconfig_reg(cs, *reg++, 0);
		}
	} else {
		assert(layout_multi == SI_PC_MULTI_ALTERNATE);

		unsigned reg_base = regs->select0;
		unsigned reg_count = count + MIN2(count, regs->num_multi);
		reg_count += regs->num_prelude;

		if (!(regs->layout & SI_PC_REG_REVERSE)) {
			radeon_set_uconfig_reg_seq(cs, reg_base, reg_count);

			for (idx = 0; idx < regs->num_prelude; ++idx)
				radeon_emit(cs, 0);
			for (idx = 0; idx < count; ++idx) {
				radeon_emit(cs, selectors[idx] | regs->select_or);
				if (idx < regs->num_multi)
					radeon_emit(cs, 0);
			}
		} else {
			reg_base -= (reg_count - 1) * 4;
			radeon_set_uconfig_reg_seq(cs, reg_base, reg_count);

			for (idx = count; idx > 0; --idx) {
				if (idx <= regs->num_multi)
					radeon_emit(cs, 0);
				radeon_emit(cs, selectors[idx - 1] | regs->select_or);
			}
			for (idx = 0; idx < regs->num_prelude; ++idx)
				radeon_emit(cs, 0);
		}
	}
}

static void si_pc_emit_start(struct si_context *sctx,
			     struct si_resource *buffer, uint64_t va)
{
	struct radeon_cmdbuf *cs = sctx->gfx_cs;

	radeon_add_to_buffer_list(sctx, sctx->gfx_cs, buffer,
				  RADEON_USAGE_WRITE, RADEON_PRIO_QUERY);

	radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
	radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) |
			COPY_DATA_DST_SEL(COPY_DATA_DST_MEM_GRBM));
	radeon_emit(cs, 1); /* immediate */
	radeon_emit(cs, 0); /* unused */
	radeon_emit(cs, va);
	radeon_emit(cs, va >> 32);

	radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
			       S_036020_PERFMON_STATE(V_036020_DISABLE_AND_RESET));
	radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
	radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_START) | EVENT_INDEX(0));
	radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
			       S_036020_PERFMON_STATE(V_036020_START_COUNTING));
}

/* Note: The buffer was already added in si_pc_emit_start, so we don't have to
 * do it again in here. */
static void si_pc_emit_stop(struct si_context *sctx,
			    struct si_resource *buffer, uint64_t va)
{
	struct radeon_cmdbuf *cs = sctx->gfx_cs;

	si_cp_release_mem(sctx, V_028A90_BOTTOM_OF_PIPE_TS, 0,
			  EOP_DST_SEL_MEM, EOP_INT_SEL_NONE,
			  EOP_DATA_SEL_VALUE_32BIT,
			  buffer, va, 0, SI_NOT_QUERY);
	si_cp_wait_mem(sctx, cs, va, 0, 0xffffffff, WAIT_REG_MEM_EQUAL);

	radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
	radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_SAMPLE) | EVENT_INDEX(0));
	radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
	radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_STOP) | EVENT_INDEX(0));
	radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
			       S_036020_PERFMON_STATE(V_036020_STOP_COUNTING) |
			       S_036020_PERFMON_SAMPLE_ENABLE(1));
}

static void si_pc_emit_read(struct si_context *sctx,
			    struct si_pc_block *block,
			    unsigned count, uint64_t va)
{
	struct si_pc_block_base *regs = block->b->b;
	struct radeon_cmdbuf *cs = sctx->gfx_cs;
	unsigned idx;
	unsigned reg = regs->counter0_lo;
	unsigned reg_delta = 8;

	if (!(regs->layout & SI_PC_FAKE)) {
		if (regs->layout & SI_PC_REG_REVERSE)
			reg_delta = -reg_delta;

		for (idx = 0; idx < count; ++idx) {
			if (regs->counters)
				reg = regs->counters[idx];

			radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
			radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) |
					COPY_DATA_DST_SEL(COPY_DATA_DST_MEM_GRBM) |
					COPY_DATA_COUNT_SEL); /* 64 bits */
			radeon_emit(cs, reg >> 2);
			radeon_emit(cs, 0); /* unused */
			radeon_emit(cs, va);
			radeon_emit(cs, va >> 32);
			va += sizeof(uint64_t);
			reg += reg_delta;
		}
	} else {
		for (idx = 0; idx < count; ++idx) {
			radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
			radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) |
					COPY_DATA_DST_SEL(COPY_DATA_DST_MEM_GRBM) |
					COPY_DATA_COUNT_SEL);
			radeon_emit(cs, 0); /* immediate */
			radeon_emit(cs, 0);
			radeon_emit(cs, va);
			radeon_emit(cs, va >> 32);
			va += sizeof(uint64_t);
		}
	}
}

static void si_pc_query_destroy(struct si_screen *sscreen,
				struct si_query *squery)
{
	struct si_query_pc *query = (struct si_query_pc *)squery;

	while (query->groups) {
		struct si_query_group *group = query->groups;
		query->groups = group->next;
		FREE(group);
	}

	FREE(query->counters);

	si_query_buffer_destroy(sscreen, &query->buffer);
	FREE(query);
}

static void si_pc_query_resume(struct si_context *sctx, struct si_query *squery)
/*
				   struct si_query_hw *hwquery,
				   struct si_resource *buffer, uint64_t va)*/
{
	struct si_query_pc *query = (struct si_query_pc *)squery;
	int current_se = -1;
	int current_instance = -1;

	if (!si_query_buffer_alloc(sctx, &query->buffer, NULL, query->result_size))
		return;
	si_need_gfx_cs_space(sctx);

	if (query->shaders)
		si_pc_emit_shaders(sctx, query->shaders);

	for (struct si_query_group *group = query->groups; group; group = group->next) {
		struct si_pc_block *block = group->block;

		if (group->se != current_se || group->instance != current_instance) {
			current_se = group->se;
			current_instance = group->instance;
			si_pc_emit_instance(sctx, group->se, group->instance);
		}

		si_pc_emit_select(sctx, block, group->num_counters, group->selectors);
	}

	if (current_se != -1 || current_instance != -1)
		si_pc_emit_instance(sctx, -1, -1);

	uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end;
	si_pc_emit_start(sctx, query->buffer.buf, va);
}

static void si_pc_query_suspend(struct si_context *sctx, struct si_query *squery)
{
	struct si_query_pc *query = (struct si_query_pc *)squery;

	if (!query->buffer.buf)
		return;

	uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end;
	query->buffer.results_end += query->result_size;

	si_pc_emit_stop(sctx, query->buffer.buf, va);

	for (struct si_query_group *group = query->groups; group; group = group->next) {
		struct si_pc_block *block = group->block;
		unsigned se = group->se >= 0 ? group->se : 0;
		unsigned se_end = se + 1;

		if ((block->b->b->flags & SI_PC_BLOCK_SE) && (group->se < 0))
			se_end = sctx->screen->info.max_se;

		do {
			unsigned instance = group->instance >= 0 ? group->instance : 0;

			do {
				si_pc_emit_instance(sctx, se, instance);
				si_pc_emit_read(sctx, block, group->num_counters, va);
				va += sizeof(uint64_t) * group->num_counters;
			} while (group->instance < 0 && ++instance < block->num_instances);
		} while (++se < se_end);
	}

	si_pc_emit_instance(sctx, -1, -1);
}

static bool si_pc_query_begin(struct si_context *ctx, struct si_query *squery)
{
	struct si_query_pc *query = (struct si_query_pc *)squery;

	si_query_buffer_reset(ctx, &query->buffer);

	LIST_ADDTAIL(&query->b.active_list, &ctx->active_queries);
	ctx->num_cs_dw_queries_suspend += query->b.num_cs_dw_suspend;

	si_pc_query_resume(ctx, squery);

	return true;
}

static bool si_pc_query_end(struct si_context *ctx, struct si_query *squery)
{
	struct si_query_pc *query = (struct si_query_pc *)squery;

	si_pc_query_suspend(ctx, squery);

	LIST_DEL(&squery->active_list);
	ctx->num_cs_dw_queries_suspend -= squery->num_cs_dw_suspend;

	return query->buffer.buf != NULL;
}

static void si_pc_query_add_result(struct si_query_pc *query,
				   void *buffer,
				   union pipe_query_result *result)
{
	uint64_t *results = buffer;
	unsigned i, j;

	for (i = 0; i < query->num_counters; ++i) {
		struct si_query_counter *counter = &query->counters[i];

		for (j = 0; j < counter->qwords; ++j) {
			uint32_t value = results[counter->base + j * counter->stride];
			result->batch[i].u64 += value;
		}
	}
}

static bool si_pc_query_get_result(struct si_context *sctx, struct si_query *squery,
				   bool wait, union pipe_query_result *result)
{
	struct si_query_pc *query = (struct si_query_pc *)squery;

	memset(result, 0, sizeof(result->batch[0]) * query->num_counters);

	for (struct si_query_buffer *qbuf = &query->buffer; qbuf; qbuf = qbuf->previous) {
		unsigned usage = PIPE_TRANSFER_READ |
				 (wait ? 0 : PIPE_TRANSFER_DONTBLOCK);
		unsigned results_base = 0;
		void *map;

		if (squery->b.flushed)
			map = sctx->ws->buffer_map(qbuf->buf->buf, NULL, usage);
		else
			map = si_buffer_map_sync_with_rings(sctx, qbuf->buf, usage);

		if (!map)
			return false;

		while (results_base != qbuf->results_end) {
			si_pc_query_add_result(query, map + results_base, result);
			results_base += query->result_size;
		}
	}

	return true;
}

static const struct si_query_ops batch_query_ops = {
	.destroy = si_pc_query_destroy,
	.begin = si_pc_query_begin,
	.end = si_pc_query_end,
	.get_result = si_pc_query_get_result,

	.suspend = si_pc_query_suspend,
	.resume = si_pc_query_resume,
};

static struct si_query_group *get_group_state(struct si_screen *screen,
					      struct si_query_pc *query,
					      struct si_pc_block *block,
					      unsigned sub_gid)
{
	struct si_query_group *group = query->groups;

	while (group) {
		if (group->block == block && group->sub_gid == sub_gid)
			return group;
		group = group->next;
	}

	group = CALLOC_STRUCT(si_query_group);
	if (!group)
		return NULL;

	group->block = block;
	group->sub_gid = sub_gid;

	if (block->b->b->flags & SI_PC_BLOCK_SHADER) {
		unsigned sub_gids = block->num_instances;
		unsigned shader_id;
		unsigned shaders;
		unsigned query_shaders;

		if (si_pc_block_has_per_se_groups(screen->perfcounters, block))
			sub_gids = sub_gids * screen->info.max_se;
		shader_id = sub_gid / sub_gids;
		sub_gid = sub_gid % sub_gids;

		shaders = si_pc_shader_type_bits[shader_id];

		query_shaders = query->shaders & ~SI_PC_SHADERS_WINDOWING;
		if (query_shaders && query_shaders != shaders) {
			fprintf(stderr, "si_perfcounter: incompatible shader groups\n");
			FREE(group);
			return NULL;
		}
		query->shaders = shaders;
	}

	if (block->b->b->flags & SI_PC_BLOCK_SHADER_WINDOWED && !query->shaders) {
		// A non-zero value in query->shaders ensures that the shader
		// masking is reset unless the user explicitly requests one.
		query->shaders = SI_PC_SHADERS_WINDOWING;
	}

	if (si_pc_block_has_per_se_groups(screen->perfcounters, block)) {
		group->se = sub_gid / block->num_instances;
		sub_gid = sub_gid % block->num_instances;
	} else {
		group->se = -1;
	}

	if (si_pc_block_has_per_instance_groups(screen->perfcounters, block)) {
		group->instance = sub_gid;
	} else {
		group->instance = -1;
	}

	group->next = query->groups;
	query->groups = group;

	return group;
}

struct pipe_query *si_create_batch_query(struct pipe_context *ctx,
					 unsigned num_queries,
					 unsigned *query_types)
{
	struct si_screen *screen =
		(struct si_screen *)ctx->screen;
	struct si_perfcounters *pc = screen->perfcounters;
	struct si_pc_block *block;
	struct si_query_group *group;
	struct si_query_pc *query;
	unsigned base_gid, sub_gid, sub_index;
	unsigned i, j;

	if (!pc)
		return NULL;

	query = CALLOC_STRUCT(si_query_pc);
	if (!query)
		return NULL;

	query->b.ops = &batch_query_ops;

	query->num_counters = num_queries;

	/* Collect selectors per group */
	for (i = 0; i < num_queries; ++i) {
		unsigned sub_gid;

		if (query_types[i] < SI_QUERY_FIRST_PERFCOUNTER)
			goto error;

		block = lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER,
				       &base_gid, &sub_index);
		if (!block)
			goto error;

		sub_gid = sub_index / block->b->selectors;
		sub_index = sub_index % block->b->selectors;

		group = get_group_state(screen, query, block, sub_gid);
		if (!group)
			goto error;

		if (group->num_counters >= block->b->b->num_counters) {
			fprintf(stderr,
				"perfcounter group %s: too many selected\n",
				block->b->b->name);
			goto error;
		}
		group->selectors[group->num_counters] = sub_index;
		++group->num_counters;
	}

	/* Compute result bases and CS size per group */
	query->b.num_cs_dw_suspend = pc->num_stop_cs_dwords;
	query->b.num_cs_dw_suspend += pc->num_instance_cs_dwords;

	i = 0;
	for (group = query->groups; group; group = group->next) {
		struct si_pc_block *block = group->block;
		unsigned read_dw;
		unsigned instances = 1;

		if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0)
			instances = screen->info.max_se;
		if (group->instance < 0)
			instances *= block->num_instances;

		group->result_base = i;
		query->result_size += sizeof(uint64_t) * instances * group->num_counters;
		i += instances * group->num_counters;

		read_dw = 6 * group->num_counters;
		query->b.num_cs_dw_suspend += instances * read_dw;
		query->b.num_cs_dw_suspend += instances * pc->num_instance_cs_dwords;
	}

	if (query->shaders) {
		if (query->shaders == SI_PC_SHADERS_WINDOWING)
			query->shaders = 0xffffffff;
	}

	/* Map user-supplied query array to result indices */
	query->counters = CALLOC(num_queries, sizeof(*query->counters));
	for (i = 0; i < num_queries; ++i) {
		struct si_query_counter *counter = &query->counters[i];
		struct si_pc_block *block;

		block = lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER,
				       &base_gid, &sub_index);

		sub_gid = sub_index / block->b->selectors;
		sub_index = sub_index % block->b->selectors;

		group = get_group_state(screen, query, block, sub_gid);
		assert(group != NULL);

		for (j = 0; j < group->num_counters; ++j) {
			if (group->selectors[j] == sub_index)
				break;
		}

		counter->base = group->result_base + j;
		counter->stride = group->num_counters;

		counter->qwords = 1;
		if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0)
			counter->qwords = screen->info.max_se;
		if (group->instance < 0)
			counter->qwords *= block->num_instances;
	}

	return (struct pipe_query *)query;

error:
	si_pc_query_destroy(screen, &query->b);
	return NULL;
}

static bool si_init_block_names(struct si_screen *screen,
				struct si_pc_block *block)
{
	bool per_instance_groups = si_pc_block_has_per_instance_groups(screen->perfcounters, block);
	bool per_se_groups = si_pc_block_has_per_se_groups(screen->perfcounters, block);
	unsigned i, j, k;
	unsigned groups_shader = 1, groups_se = 1, groups_instance = 1;
	unsigned namelen;
	char *groupname;
	char *p;

	if (per_instance_groups)
		groups_instance = block->num_instances;
	if (per_se_groups)
		groups_se = screen->info.max_se;
	if (block->b->b->flags & SI_PC_BLOCK_SHADER)
		groups_shader = ARRAY_SIZE(si_pc_shader_type_bits);

	namelen = strlen(block->b->b->name);
	block->group_name_stride = namelen + 1;
	if (block->b->b->flags & SI_PC_BLOCK_SHADER)
		block->group_name_stride += 3;
	if (per_se_groups) {
		assert(groups_se <= 10);
		block->group_name_stride += 1;

		if (per_instance_groups)
			block->group_name_stride += 1;
	}
	if (per_instance_groups) {
		assert(groups_instance <= 100);
		block->group_name_stride += 2;
	}

	block->group_names = MALLOC(block->num_groups * block->group_name_stride);
	if (!block->group_names)
		return false;

	groupname = block->group_names;
	for (i = 0; i < groups_shader; ++i) {
		const char *shader_suffix = si_pc_shader_type_suffixes[i];
		unsigned shaderlen = strlen(shader_suffix);
		for (j = 0; j < groups_se; ++j) {
			for (k = 0; k < groups_instance; ++k) {
				strcpy(groupname, block->b->b->name);
				p = groupname + namelen;

				if (block->b->b->flags & SI_PC_BLOCK_SHADER) {
					strcpy(p, shader_suffix);
					p += shaderlen;
				}

				if (per_se_groups) {
					p += sprintf(p, "%d", j);
					if (per_instance_groups)
						*p++ = '_';
				}

				if (per_instance_groups)
					p += sprintf(p, "%d", k);

				groupname += block->group_name_stride;
			}
		}
	}

	assert(block->b->selectors <= 1000);
	block->selector_name_stride = block->group_name_stride + 4;
	block->selector_names = MALLOC(block->num_groups * block->b->selectors *
				       block->selector_name_stride);
	if (!block->selector_names)
		return false;

	groupname = block->group_names;
	p = block->selector_names;
	for (i = 0; i < block->num_groups; ++i) {
		for (j = 0; j < block->b->selectors; ++j) {
			sprintf(p, "%s_%03d", groupname, j);
			p += block->selector_name_stride;
		}
		groupname += block->group_name_stride;
	}

	return true;
}

int si_get_perfcounter_info(struct si_screen *screen,
			    unsigned index,
			    struct pipe_driver_query_info *info)
{
	struct si_perfcounters *pc = screen->perfcounters;
	struct si_pc_block *block;
	unsigned base_gid, sub;

	if (!pc)
		return 0;

	if (!info) {
		unsigned bid, num_queries = 0;

		for (bid = 0; bid < pc->num_blocks; ++bid) {
			num_queries += pc->blocks[bid].b->selectors *
				       pc->blocks[bid].num_groups;
		}

		return num_queries;
	}

	block = lookup_counter(pc, index, &base_gid, &sub);
	if (!block)
		return 0;

	if (!block->selector_names) {
		if (!si_init_block_names(screen, block))
			return 0;
	}
	info->name = block->selector_names + sub * block->selector_name_stride;
	info->query_type = SI_QUERY_FIRST_PERFCOUNTER + index;
	info->max_value.u64 = 0;
	info->type = PIPE_DRIVER_QUERY_TYPE_UINT64;
	info->result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_AVERAGE;
	info->group_id = base_gid + sub / block->b->selectors;
	info->flags = PIPE_DRIVER_QUERY_FLAG_BATCH;
	if (sub > 0 && sub + 1 < block->b->selectors * block->num_groups)
		info->flags |= PIPE_DRIVER_QUERY_FLAG_DONT_LIST;
	return 1;
}

int si_get_perfcounter_group_info(struct si_screen *screen,
				  unsigned index,
				  struct pipe_driver_query_group_info *info)
{
	struct si_perfcounters *pc = screen->perfcounters;
	struct si_pc_block *block;

	if (!pc)
		return 0;

	if (!info)
		return pc->num_groups;

	block = lookup_group(pc, &index);
	if (!block)
		return 0;

	if (!block->group_names) {
		if (!si_init_block_names(screen, block))
			return 0;
	}
	info->name = block->group_names + index * block->group_name_stride;
	info->num_queries = block->b->selectors;
	info->max_active_queries = block->b->b->num_counters;
	return 1;
}

void si_destroy_perfcounters(struct si_screen *screen)
{
	struct si_perfcounters *pc = screen->perfcounters;
	unsigned i;

	if (!pc)
		return;

	for (i = 0; i < pc->num_blocks; ++i) {
		FREE(pc->blocks[i].group_names);
		FREE(pc->blocks[i].selector_names);
	}
	FREE(pc->blocks);
	FREE(pc);
	screen->perfcounters = NULL;
}

void si_init_perfcounters(struct si_screen *screen)
{
	struct si_perfcounters *pc;
	const struct si_pc_block_gfxdescr *blocks;
	unsigned num_blocks;
	unsigned i;

	switch (screen->info.chip_class) {
	case CIK:
		blocks = groups_CIK;
		num_blocks = ARRAY_SIZE(groups_CIK);
		break;
	case VI:
		blocks = groups_VI;
		num_blocks = ARRAY_SIZE(groups_VI);
		break;
	case GFX9:
		blocks = groups_gfx9;
		num_blocks = ARRAY_SIZE(groups_gfx9);
		break;
	case SI:
	default:
		return; /* not implemented */
	}

	if (screen->info.max_sh_per_se != 1) {
		/* This should not happen on non-SI chips. */
		fprintf(stderr, "si_init_perfcounters: max_sh_per_se = %d not "
			"supported (inaccurate performance counters)\n",
			screen->info.max_sh_per_se);
	}

	screen->perfcounters = pc = CALLOC_STRUCT(si_perfcounters);
	if (!pc)
		return;

	pc->num_stop_cs_dwords = 14 + si_cp_write_fence_dwords(screen);
	pc->num_instance_cs_dwords = 3;

	pc->separate_se = debug_get_bool_option("RADEON_PC_SEPARATE_SE", false);
	pc->separate_instance = debug_get_bool_option("RADEON_PC_SEPARATE_INSTANCE", false);

	pc->blocks = CALLOC(num_blocks, sizeof(struct si_pc_block));
	if (!pc->blocks)
		goto error;
	pc->num_blocks = num_blocks;

	for (i = 0; i < num_blocks; ++i) {
		struct si_pc_block *block = &pc->blocks[i];
		block->b = &blocks[i];
		block->num_instances = block->b->instances;

		if (!strcmp(block->b->b->name, "CB") ||
		    !strcmp(block->b->b->name, "DB"))
			block->num_instances = screen->info.max_se;
		else if (!strcmp(block->b->b->name, "TCC"))
			block->num_instances = screen->info.num_tcc_blocks;
		else if (!strcmp(block->b->b->name, "IA"))
			block->num_instances = MAX2(1, screen->info.max_se / 2);

		if (si_pc_block_has_per_instance_groups(pc, block)) {
			block->num_groups = block->num_instances;
		} else {
			block->num_groups = 1;
		}

		if (si_pc_block_has_per_se_groups(pc, block))
			block->num_groups *= screen->info.max_se;
		if (block->b->b->flags & SI_PC_BLOCK_SHADER)
			block->num_groups *= ARRAY_SIZE(si_pc_shader_type_bits);

		pc->num_groups += block->num_groups;
	}

	return;

error:
	si_destroy_perfcounters(screen);
}