r600: fork and import gallium/radeon

This marks the end of code sharing between r600 and radeonsi.
It's getting difficult to work on radeonsi without breaking r600.

A lot of functions had to be renamed to prevent linker conflicts.
There are also minor cleanups.

Acked-by: Dave Airlie <[email protected]>
Reviewed-by: Nicolai Hähnle <[email protected]>

25 files changed, 14508 insertions, 12 deletions

diff --git a/src/gallium/drivers/r600/Automake.inc b/src/gallium/drivers/r600/Automake.inc
index bb9f6ec8f53..727b1174d3f 100644
--- a/src/gallium/drivers/r600/Automake.inc
+++ b/src/gallium/drivers/r600/Automake.inc
@@ -11,6 +11,4 @@ TARGET_LIB_DEPS += \
 TARGET_RADEON_WINSYS = \
 	$(top_builddir)/src/gallium/winsys/radeon/drm/libradeonwinsys.la
 
-TARGET_RADEON_COMMON = \
-	$(top_builddir)/src/gallium/drivers/radeon/libradeon.la
 endif
diff --git a/src/gallium/drivers/r600/Makefile.am b/src/gallium/drivers/r600/Makefile.am
index 36d240d1566..6cb9307ade7 100644
--- a/src/gallium/drivers/r600/Makefile.am
+++ b/src/gallium/drivers/r600/Makefile.am
@@ -27,8 +27,7 @@ libr600_la_SOURCES = \
 if HAVE_GALLIUM_LLVM
 
 AM_CFLAGS += \
-	$(LLVM_CFLAGS) \
-	-I$(top_srcdir)/src/gallium/drivers/radeon/
+	$(LLVM_CFLAGS)
 
 endif
 
diff --git a/src/gallium/drivers/r600/Makefile.sources b/src/gallium/drivers/r600/Makefile.sources
index 2f206520fa7..c232d6ac938 100644
--- a/src/gallium/drivers/r600/Makefile.sources
+++ b/src/gallium/drivers/r600/Makefile.sources
@@ -29,7 +29,26 @@ C_SOURCES = \
 	r600_state_common.c \
 	r600_uvd.c \
 	r700_asm.c \
-	r700_sq.h
+	r700_sq.h \
+	cayman_msaa.c \
+	r600_buffer_common.c \
+	r600_cs.h \
+	r600_gpu_load.c \
+	r600_perfcounter.c \
+	r600_pipe_common.c \
+	r600_pipe_common.h \
+	r600_query.c \
+	r600_query.h \
+	r600_streamout.c \
+	r600_test_dma.c \
+	r600_texture.c \
+	r600_viewport.c \
+	radeon_uvd.c \
+	radeon_uvd.h \
+	radeon_vce.c \
+	radeon_vce.h \
+	radeon_video.c \
+	radeon_video.h
 
 CXX_SOURCES = \
 	sb/sb_bc_builder.cpp \
diff --git a/src/gallium/drivers/r600/cayman_msaa.c b/src/gallium/drivers/r600/cayman_msaa.c
new file mode 100644
index 00000000000..33f1040185a
--- /dev/null
+++ b/src/gallium/drivers/r600/cayman_msaa.c
@@ -0,0 +1,269 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors: Marek Olšák <[email protected]>
+ *
+ */
+
+#include "r600_cs.h"
+
+/* 2xMSAA
+ * There are two locations (4, 4), (-4, -4). */
+const uint32_t eg_sample_locs_2x[4] = {
+	FILL_SREG(4, 4, -4, -4, 4, 4, -4, -4),
+	FILL_SREG(4, 4, -4, -4, 4, 4, -4, -4),
+	FILL_SREG(4, 4, -4, -4, 4, 4, -4, -4),
+	FILL_SREG(4, 4, -4, -4, 4, 4, -4, -4),
+};
+const unsigned eg_max_dist_2x = 4;
+/* 4xMSAA
+ * There are 4 locations: (-2, 6), (6, -2), (-6, 2), (2, 6). */
+const uint32_t eg_sample_locs_4x[4] = {
+	FILL_SREG(-2, -6, 6, -2, -6, 2, 2, 6),
+	FILL_SREG(-2, -6, 6, -2, -6, 2, 2, 6),
+	FILL_SREG(-2, -6, 6, -2, -6, 2, 2, 6),
+	FILL_SREG(-2, -6, 6, -2, -6, 2, 2, 6),
+};
+const unsigned eg_max_dist_4x = 6;
+
+/* Cayman 8xMSAA */
+static const uint32_t cm_sample_locs_8x[] = {
+	FILL_SREG( 1, -3, -1,  3, 5,  1, -3, -5),
+	FILL_SREG( 1, -3, -1,  3, 5,  1, -3, -5),
+	FILL_SREG( 1, -3, -1,  3, 5,  1, -3, -5),
+	FILL_SREG( 1, -3, -1,  3, 5,  1, -3, -5),
+	FILL_SREG(-5,  5, -7, -1, 3,  7,  7, -7),
+	FILL_SREG(-5,  5, -7, -1, 3,  7,  7, -7),
+	FILL_SREG(-5,  5, -7, -1, 3,  7,  7, -7),
+	FILL_SREG(-5,  5, -7, -1, 3,  7,  7, -7),
+};
+static const unsigned cm_max_dist_8x = 8;
+/* Cayman 16xMSAA */
+static const uint32_t cm_sample_locs_16x[] = {
+	FILL_SREG( 1,  1, -1, -3, -3,  2,  4, -1),
+	FILL_SREG( 1,  1, -1, -3, -3,  2,  4, -1),
+	FILL_SREG( 1,  1, -1, -3, -3,  2,  4, -1),
+	FILL_SREG( 1,  1, -1, -3, -3,  2,  4, -1),
+	FILL_SREG(-5, -2,  2,  5,  5,  3,  3, -5),
+	FILL_SREG(-5, -2,  2,  5,  5,  3,  3, -5),
+	FILL_SREG(-5, -2,  2,  5,  5,  3,  3, -5),
+	FILL_SREG(-5, -2,  2,  5,  5,  3,  3, -5),
+	FILL_SREG(-2,  6,  0, -7, -4, -6, -6,  4),
+	FILL_SREG(-2,  6,  0, -7, -4, -6, -6,  4),
+	FILL_SREG(-2,  6,  0, -7, -4, -6, -6,  4),
+	FILL_SREG(-2,  6,  0, -7, -4, -6, -6,  4),
+	FILL_SREG(-8,  0,  7, -4,  6,  7, -7, -8),
+	FILL_SREG(-8,  0,  7, -4,  6,  7, -7, -8),
+	FILL_SREG(-8,  0,  7, -4,  6,  7, -7, -8),
+	FILL_SREG(-8,  0,  7, -4,  6,  7, -7, -8),
+};
+static const unsigned cm_max_dist_16x = 8;
+
+void cayman_get_sample_position(struct pipe_context *ctx, unsigned sample_count,
+				unsigned sample_index, float *out_value)
+{
+	int offset, index;
+	struct {
+		int idx:4;
+	} val;
+	switch (sample_count) {
+	case 1:
+	default:
+		out_value[0] = out_value[1] = 0.5;
+		break;
+	case 2:
+		offset = 4 * (sample_index * 2);
+		val.idx = (eg_sample_locs_2x[0] >> offset) & 0xf;
+		out_value[0] = (float)(val.idx + 8) / 16.0f;
+		val.idx = (eg_sample_locs_2x[0] >> (offset + 4)) & 0xf;
+		out_value[1] = (float)(val.idx + 8) / 16.0f;
+		break;
+	case 4:
+		offset = 4 * (sample_index * 2);
+		val.idx = (eg_sample_locs_4x[0] >> offset) & 0xf;
+		out_value[0] = (float)(val.idx + 8) / 16.0f;
+		val.idx = (eg_sample_locs_4x[0] >> (offset + 4)) & 0xf;
+		out_value[1] = (float)(val.idx + 8) / 16.0f;
+		break;
+	case 8:
+		offset = 4 * (sample_index % 4 * 2);
+		index = (sample_index / 4) * 4;
+		val.idx = (cm_sample_locs_8x[index] >> offset) & 0xf;
+		out_value[0] = (float)(val.idx + 8) / 16.0f;
+		val.idx = (cm_sample_locs_8x[index] >> (offset + 4)) & 0xf;
+		out_value[1] = (float)(val.idx + 8) / 16.0f;
+		break;
+	case 16:
+		offset = 4 * (sample_index % 4 * 2);
+		index = (sample_index / 4) * 4;
+		val.idx = (cm_sample_locs_16x[index] >> offset) & 0xf;
+		out_value[0] = (float)(val.idx + 8) / 16.0f;
+		val.idx = (cm_sample_locs_16x[index] >> (offset + 4)) & 0xf;
+		out_value[1] = (float)(val.idx + 8) / 16.0f;
+		break;
+	}
+}
+
+void cayman_init_msaa(struct pipe_context *ctx)
+{
+	struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+	int i;
+
+	cayman_get_sample_position(ctx, 1, 0, rctx->sample_locations_1x[0]);
+
+	for (i = 0; i < 2; i++)
+		cayman_get_sample_position(ctx, 2, i, rctx->sample_locations_2x[i]);
+	for (i = 0; i < 4; i++)
+		cayman_get_sample_position(ctx, 4, i, rctx->sample_locations_4x[i]);
+	for (i = 0; i < 8; i++)
+		cayman_get_sample_position(ctx, 8, i, rctx->sample_locations_8x[i]);
+	for (i = 0; i < 16; i++)
+		cayman_get_sample_position(ctx, 16, i, rctx->sample_locations_16x[i]);
+}
+
+void cayman_emit_msaa_sample_locs(struct radeon_winsys_cs *cs, int nr_samples)
+{
+	switch (nr_samples) {
+	default:
+	case 1:
+		radeon_set_context_reg(cs, CM_R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, 0);
+		radeon_set_context_reg(cs, CM_R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, 0);
+		radeon_set_context_reg(cs, CM_R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, 0);
+		radeon_set_context_reg(cs, CM_R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, 0);
+		break;
+	case 2:
+		radeon_set_context_reg(cs, CM_R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, eg_sample_locs_2x[0]);
+		radeon_set_context_reg(cs, CM_R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, eg_sample_locs_2x[1]);
+		radeon_set_context_reg(cs, CM_R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, eg_sample_locs_2x[2]);
+		radeon_set_context_reg(cs, CM_R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, eg_sample_locs_2x[3]);
+		break;
+	case 4:
+		radeon_set_context_reg(cs, CM_R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, eg_sample_locs_4x[0]);
+		radeon_set_context_reg(cs, CM_R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, eg_sample_locs_4x[1]);
+		radeon_set_context_reg(cs, CM_R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, eg_sample_locs_4x[2]);
+		radeon_set_context_reg(cs, CM_R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, eg_sample_locs_4x[3]);
+		break;
+	case 8:
+		radeon_set_context_reg_seq(cs, CM_R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, 14);
+		radeon_emit(cs, cm_sample_locs_8x[0]);
+		radeon_emit(cs, cm_sample_locs_8x[4]);
+		radeon_emit(cs, 0);
+		radeon_emit(cs, 0);
+		radeon_emit(cs, cm_sample_locs_8x[1]);
+		radeon_emit(cs, cm_sample_locs_8x[5]);
+		radeon_emit(cs, 0);
+		radeon_emit(cs, 0);
+		radeon_emit(cs, cm_sample_locs_8x[2]);
+		radeon_emit(cs, cm_sample_locs_8x[6]);
+		radeon_emit(cs, 0);
+		radeon_emit(cs, 0);
+		radeon_emit(cs, cm_sample_locs_8x[3]);
+		radeon_emit(cs, cm_sample_locs_8x[7]);
+		break;
+	case 16:
+		radeon_set_context_reg_seq(cs, CM_R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, 16);
+		radeon_emit(cs, cm_sample_locs_16x[0]);
+		radeon_emit(cs, cm_sample_locs_16x[4]);
+		radeon_emit(cs, cm_sample_locs_16x[8]);
+		radeon_emit(cs, cm_sample_locs_16x[12]);
+		radeon_emit(cs, cm_sample_locs_16x[1]);
+		radeon_emit(cs, cm_sample_locs_16x[5]);
+		radeon_emit(cs, cm_sample_locs_16x[9]);
+		radeon_emit(cs, cm_sample_locs_16x[13]);
+		radeon_emit(cs, cm_sample_locs_16x[2]);
+		radeon_emit(cs, cm_sample_locs_16x[6]);
+		radeon_emit(cs, cm_sample_locs_16x[10]);
+		radeon_emit(cs, cm_sample_locs_16x[14]);
+		radeon_emit(cs, cm_sample_locs_16x[3]);
+		radeon_emit(cs, cm_sample_locs_16x[7]);
+		radeon_emit(cs, cm_sample_locs_16x[11]);
+		radeon_emit(cs, cm_sample_locs_16x[15]);
+		break;
+	}
+}
+
+void cayman_emit_msaa_config(struct radeon_winsys_cs *cs, int nr_samples,
+			     int ps_iter_samples, int overrast_samples,
+			     unsigned sc_mode_cntl_1)
+{
+	int setup_samples = nr_samples > 1 ? nr_samples :
+			    overrast_samples > 1 ? overrast_samples : 0;
+	/* Required by OpenGL line rasterization.
+	 *
+	 * TODO: We should also enable perpendicular endcaps for AA lines,
+	 *       but that requires implementing line stippling in the pixel
+	 *       shader. SC can only do line stippling with axis-aligned
+	 *       endcaps.
+	 */
+	unsigned sc_line_cntl = S_028BDC_DX10_DIAMOND_TEST_ENA(1);
+
+	if (setup_samples > 1) {
+		/* indexed by log2(nr_samples) */
+		unsigned max_dist[] = {
+			0,
+			eg_max_dist_2x,
+			eg_max_dist_4x,
+			cm_max_dist_8x,
+			cm_max_dist_16x
+		};
+		unsigned log_samples = util_logbase2(setup_samples);
+		unsigned log_ps_iter_samples =
+			util_logbase2(util_next_power_of_two(ps_iter_samples));
+
+		radeon_set_context_reg_seq(cs, CM_R_028BDC_PA_SC_LINE_CNTL, 2);
+		radeon_emit(cs, sc_line_cntl |
+			    S_028BDC_EXPAND_LINE_WIDTH(1)); /* CM_R_028BDC_PA_SC_LINE_CNTL */
+		radeon_emit(cs, S_028BE0_MSAA_NUM_SAMPLES(log_samples) |
+			    S_028BE0_MAX_SAMPLE_DIST(max_dist[log_samples]) |
+			    S_028BE0_MSAA_EXPOSED_SAMPLES(log_samples)); /* CM_R_028BE0_PA_SC_AA_CONFIG */
+
+		if (nr_samples > 1) {
+			radeon_set_context_reg(cs, CM_R_028804_DB_EQAA,
+					       S_028804_MAX_ANCHOR_SAMPLES(log_samples) |
+					       S_028804_PS_ITER_SAMPLES(log_ps_iter_samples) |
+					       S_028804_MASK_EXPORT_NUM_SAMPLES(log_samples) |
+					       S_028804_ALPHA_TO_MASK_NUM_SAMPLES(log_samples) |
+					       S_028804_HIGH_QUALITY_INTERSECTIONS(1) |
+					       S_028804_STATIC_ANCHOR_ASSOCIATIONS(1));
+			radeon_set_context_reg(cs, EG_R_028A4C_PA_SC_MODE_CNTL_1,
+					       EG_S_028A4C_PS_ITER_SAMPLE(ps_iter_samples > 1) |
+					       sc_mode_cntl_1);
+		} else if (overrast_samples > 1) {
+			radeon_set_context_reg(cs, CM_R_028804_DB_EQAA,
+					       S_028804_HIGH_QUALITY_INTERSECTIONS(1) |
+					       S_028804_STATIC_ANCHOR_ASSOCIATIONS(1) |
+					       S_028804_OVERRASTERIZATION_AMOUNT(log_samples));
+			radeon_set_context_reg(cs, EG_R_028A4C_PA_SC_MODE_CNTL_1,
+					       sc_mode_cntl_1);
+		}
+	} else {
+		radeon_set_context_reg_seq(cs, CM_R_028BDC_PA_SC_LINE_CNTL, 2);
+		radeon_emit(cs, sc_line_cntl); /* CM_R_028BDC_PA_SC_LINE_CNTL */
+		radeon_emit(cs, 0); /* CM_R_028BE0_PA_SC_AA_CONFIG */
+
+		radeon_set_context_reg(cs, CM_R_028804_DB_EQAA,
+				       S_028804_HIGH_QUALITY_INTERSECTIONS(1) |
+				       S_028804_STATIC_ANCHOR_ASSOCIATIONS(1));
+		radeon_set_context_reg(cs, EG_R_028A4C_PA_SC_MODE_CNTL_1,
+				       sc_mode_cntl_1);
+	}
+}
diff --git a/src/gallium/drivers/r600/r600_buffer_common.c b/src/gallium/drivers/r600/r600_buffer_common.c
new file mode 100644
index 00000000000..f35bc2c813b
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_buffer_common.c
@@ -0,0 +1,687 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *      Marek Olšák
+ */
+
+#include "r600_cs.h"
+#include "util/u_memory.h"
+#include "util/u_upload_mgr.h"
+#include <inttypes.h>
+#include <stdio.h>
+
+bool r600_rings_is_buffer_referenced(struct r600_common_context *ctx,
+				     struct pb_buffer *buf,
+				     enum radeon_bo_usage usage)
+{
+	if (ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, buf, usage)) {
+		return true;
+	}
+	if (radeon_emitted(ctx->dma.cs, 0) &&
+	    ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, buf, usage)) {
+		return true;
+	}
+	return false;
+}
+
+void *r600_buffer_map_sync_with_rings(struct r600_common_context *ctx,
+                                      struct r600_resource *resource,
+                                      unsigned usage)
+{
+	enum radeon_bo_usage rusage = RADEON_USAGE_READWRITE;
+	bool busy = false;
+
+	assert(!(resource->flags & RADEON_FLAG_SPARSE));
+
+	if (usage & PIPE_TRANSFER_UNSYNCHRONIZED) {
+		return ctx->ws->buffer_map(resource->buf, NULL, usage);
+	}
+
+	if (!(usage & PIPE_TRANSFER_WRITE)) {
+		/* have to wait for the last write */
+		rusage = RADEON_USAGE_WRITE;
+	}
+
+	if (radeon_emitted(ctx->gfx.cs, ctx->initial_gfx_cs_size) &&
+	    ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs,
+					     resource->buf, rusage)) {
+		if (usage & PIPE_TRANSFER_DONTBLOCK) {
+			ctx->gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
+			return NULL;
+		} else {
+			ctx->gfx.flush(ctx, 0, NULL);
+			busy = true;
+		}
+	}
+	if (radeon_emitted(ctx->dma.cs, 0) &&
+	    ctx->ws->cs_is_buffer_referenced(ctx->dma.cs,
+					     resource->buf, rusage)) {
+		if (usage & PIPE_TRANSFER_DONTBLOCK) {
+			ctx->dma.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
+			return NULL;
+		} else {
+			ctx->dma.flush(ctx, 0, NULL);
+			busy = true;
+		}
+	}
+
+	if (busy || !ctx->ws->buffer_wait(resource->buf, 0, rusage)) {
+		if (usage & PIPE_TRANSFER_DONTBLOCK) {
+			return NULL;
+		} else {
+			/* We will be wait for the GPU. Wait for any offloaded
+			 * CS flush to complete to avoid busy-waiting in the winsys. */
+			ctx->ws->cs_sync_flush(ctx->gfx.cs);
+			if (ctx->dma.cs)
+				ctx->ws->cs_sync_flush(ctx->dma.cs);
+		}
+	}
+
+	/* Setting the CS to NULL will prevent doing checks we have done already. */
+	return ctx->ws->buffer_map(resource->buf, NULL, usage);
+}
+
+void r600_init_resource_fields(struct r600_common_screen *rscreen,
+			       struct r600_resource *res,
+			       uint64_t size, unsigned alignment)
+{
+	struct r600_texture *rtex = (struct r600_texture*)res;
+
+	res->bo_size = size;
+	res->bo_alignment = alignment;
+	res->flags = 0;
+	res->texture_handle_allocated = false;
+	res->image_handle_allocated = false;
+
+	switch (res->b.b.usage) {
+	case PIPE_USAGE_STREAM:
+		res->flags = RADEON_FLAG_GTT_WC;
+		/* fall through */
+	case PIPE_USAGE_STAGING:
+		/* Transfers are likely to occur more often with these
+		 * resources. */
+		res->domains = RADEON_DOMAIN_GTT;
+		break;
+	case PIPE_USAGE_DYNAMIC:
+		/* Older kernels didn't always flush the HDP cache before
+		 * CS execution
+		 */
+		if (rscreen->info.drm_major == 2 &&
+		    rscreen->info.drm_minor < 40) {
+			res->domains = RADEON_DOMAIN_GTT;
+			res->flags |= RADEON_FLAG_GTT_WC;
+			break;
+		}
+		/* fall through */
+	case PIPE_USAGE_DEFAULT:
+	case PIPE_USAGE_IMMUTABLE:
+	default:
+		/* Not listing GTT here improves performance in some
+		 * apps. */
+		res->domains = RADEON_DOMAIN_VRAM;
+		res->flags |= RADEON_FLAG_GTT_WC;
+		break;
+	}
+
+	if (res->b.b.target == PIPE_BUFFER &&
+	    res->b.b.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
+			      PIPE_RESOURCE_FLAG_MAP_COHERENT)) {
+		/* Use GTT for all persistent mappings with older
+		 * kernels, because they didn't always flush the HDP
+		 * cache before CS execution.
+		 *
+		 * Write-combined CPU mappings are fine, the kernel
+		 * ensures all CPU writes finish before the GPU
+		 * executes a command stream.
+		 */
+		if (rscreen->info.drm_major == 2 &&
+		    rscreen->info.drm_minor < 40)
+			res->domains = RADEON_DOMAIN_GTT;
+	}
+
+	/* Tiled textures are unmappable. Always put them in VRAM. */
+	if ((res->b.b.target != PIPE_BUFFER && !rtex->surface.is_linear) ||
+	    res->flags & R600_RESOURCE_FLAG_UNMAPPABLE) {
+		res->domains = RADEON_DOMAIN_VRAM;
+		res->flags |= RADEON_FLAG_NO_CPU_ACCESS |
+			 RADEON_FLAG_GTT_WC;
+	}
+
+	/* Only displayable single-sample textures can be shared between
+	 * processes. */
+	if (res->b.b.target == PIPE_BUFFER ||
+	    res->b.b.nr_samples >= 2 ||
+	    (rtex->surface.micro_tile_mode != RADEON_MICRO_MODE_DISPLAY &&
+	     /* Raven doesn't use display micro mode for 32bpp, so check this: */
+	     !(res->b.b.bind & PIPE_BIND_SCANOUT)))
+		res->flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING;
+
+	/* If VRAM is just stolen system memory, allow both VRAM and
+	 * GTT, whichever has free space. If a buffer is evicted from
+	 * VRAM to GTT, it will stay there.
+	 *
+	 * DRM 3.6.0 has good BO move throttling, so we can allow VRAM-only
+	 * placements even with a low amount of stolen VRAM.
+	 */
+	if (!rscreen->info.has_dedicated_vram &&
+	    (rscreen->info.drm_major < 3 || rscreen->info.drm_minor < 6) &&
+	    res->domains == RADEON_DOMAIN_VRAM) {
+		res->domains = RADEON_DOMAIN_VRAM_GTT;
+		res->flags &= ~RADEON_FLAG_NO_CPU_ACCESS; /* disallowed with VRAM_GTT */
+	}
+
+	if (rscreen->debug_flags & DBG_NO_WC)
+		res->flags &= ~RADEON_FLAG_GTT_WC;
+
+	if (res->b.b.bind & PIPE_BIND_SHARED)
+		res->flags |= RADEON_FLAG_NO_SUBALLOC;
+
+	/* Set expected VRAM and GART usage for the buffer. */
+	res->vram_usage = 0;
+	res->gart_usage = 0;
+
+	if (res->domains & RADEON_DOMAIN_VRAM)
+		res->vram_usage = size;
+	else if (res->domains & RADEON_DOMAIN_GTT)
+		res->gart_usage = size;
+}
+
+bool r600_alloc_resource(struct r600_common_screen *rscreen,
+			 struct r600_resource *res)
+{
+	struct pb_buffer *old_buf, *new_buf;
+
+	/* Allocate a new resource. */
+	new_buf = rscreen->ws->buffer_create(rscreen->ws, res->bo_size,
+					     res->bo_alignment,
+					     res->domains, res->flags);
+	if (!new_buf) {
+		return false;
+	}
+
+	/* Replace the pointer such that if res->buf wasn't NULL, it won't be
+	 * NULL. This should prevent crashes with multiple contexts using
+	 * the same buffer where one of the contexts invalidates it while
+	 * the others are using it. */
+	old_buf = res->buf;
+	res->buf = new_buf; /* should be atomic */
+
+	if (rscreen->info.has_virtual_memory)
+		res->gpu_address = rscreen->ws->buffer_get_virtual_address(res->buf);
+	else
+		res->gpu_address = 0;
+
+	pb_reference(&old_buf, NULL);
+
+	util_range_set_empty(&res->valid_buffer_range);
+	res->TC_L2_dirty = false;
+
+	/* Print debug information. */
+	if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) {
+		fprintf(stderr, "VM start=0x%"PRIX64"  end=0x%"PRIX64" | Buffer %"PRIu64" bytes\n",
+			res->gpu_address, res->gpu_address + res->buf->size,
+			res->buf->size);
+	}
+	return true;
+}
+
+static void r600_buffer_destroy(struct pipe_screen *screen,
+				struct pipe_resource *buf)
+{
+	struct r600_resource *rbuffer = r600_resource(buf);
+
+	threaded_resource_deinit(buf);
+	util_range_destroy(&rbuffer->valid_buffer_range);
+	pb_reference(&rbuffer->buf, NULL);
+	FREE(rbuffer);
+}
+
+static bool
+r600_invalidate_buffer(struct r600_common_context *rctx,
+		       struct r600_resource *rbuffer)
+{
+	/* Shared buffers can't be reallocated. */
+	if (rbuffer->b.is_shared)
+		return false;
+
+	/* Sparse buffers can't be reallocated. */
+	if (rbuffer->flags & RADEON_FLAG_SPARSE)
+		return false;
+
+	/* In AMD_pinned_memory, the user pointer association only gets
+	 * broken when the buffer is explicitly re-allocated.
+	 */
+	if (rbuffer->b.is_user_ptr)
+		return false;
+
+	/* Check if mapping this buffer would cause waiting for the GPU. */
+	if (r600_rings_is_buffer_referenced(rctx, rbuffer->buf, RADEON_USAGE_READWRITE) ||
+	    !rctx->ws->buffer_wait(rbuffer->buf, 0, RADEON_USAGE_READWRITE)) {
+		rctx->invalidate_buffer(&rctx->b, &rbuffer->b.b);
+	} else {
+		util_range_set_empty(&rbuffer->valid_buffer_range);
+	}
+
+	return true;
+}
+
+/* Replace the storage of dst with src. */
+void r600_replace_buffer_storage(struct pipe_context *ctx,
+				 struct pipe_resource *dst,
+				 struct pipe_resource *src)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct r600_resource *rdst = r600_resource(dst);
+	struct r600_resource *rsrc = r600_resource(src);
+	uint64_t old_gpu_address = rdst->gpu_address;
+
+	pb_reference(&rdst->buf, rsrc->buf);
+	rdst->gpu_address = rsrc->gpu_address;
+	rdst->b.b.bind = rsrc->b.b.bind;
+	rdst->flags = rsrc->flags;
+
+	assert(rdst->vram_usage == rsrc->vram_usage);
+	assert(rdst->gart_usage == rsrc->gart_usage);
+	assert(rdst->bo_size == rsrc->bo_size);
+	assert(rdst->bo_alignment == rsrc->bo_alignment);
+	assert(rdst->domains == rsrc->domains);
+
+	rctx->rebind_buffer(ctx, dst, old_gpu_address);
+}
+
+void r600_invalidate_resource(struct pipe_context *ctx,
+			      struct pipe_resource *resource)
+{
+	struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+	struct r600_resource *rbuffer = r600_resource(resource);
+
+	/* We currently only do anyting here for buffers */
+	if (resource->target == PIPE_BUFFER)
+		(void)r600_invalidate_buffer(rctx, rbuffer);
+}
+
+static void *r600_buffer_get_transfer(struct pipe_context *ctx,
+				      struct pipe_resource *resource,
+                                      unsigned usage,
+                                      const struct pipe_box *box,
+				      struct pipe_transfer **ptransfer,
+				      void *data, struct r600_resource *staging,
+				      unsigned offset)
+{
+	struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+	struct r600_transfer *transfer;
+
+	if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC)
+		transfer = slab_alloc(&rctx->pool_transfers_unsync);
+	else
+		transfer = slab_alloc(&rctx->pool_transfers);
+
+	transfer->b.b.resource = NULL;
+	pipe_resource_reference(&transfer->b.b.resource, resource);
+	transfer->b.b.level = 0;
+	transfer->b.b.usage = usage;
+	transfer->b.b.box = *box;
+	transfer->b.b.stride = 0;
+	transfer->b.b.layer_stride = 0;
+	transfer->b.staging = NULL;
+	transfer->offset = offset;
+	transfer->staging = staging;
+	*ptransfer = &transfer->b.b;
+	return data;
+}
+
+static bool r600_can_dma_copy_buffer(struct r600_common_context *rctx,
+				     unsigned dstx, unsigned srcx, unsigned size)
+{
+	bool dword_aligned = !(dstx % 4) && !(srcx % 4) && !(size % 4);
+
+	return rctx->screen->has_cp_dma ||
+	       (dword_aligned && (rctx->dma.cs ||
+				  rctx->screen->has_streamout));
+
+}
+
+static void *r600_buffer_transfer_map(struct pipe_context *ctx,
+                                      struct pipe_resource *resource,
+                                      unsigned level,
+                                      unsigned usage,
+                                      const struct pipe_box *box,
+                                      struct pipe_transfer **ptransfer)
+{
+	struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)ctx->screen;
+	struct r600_resource *rbuffer = r600_resource(resource);
+	uint8_t *data;
+
+	assert(box->x + box->width <= resource->width0);
+
+	/* From GL_AMD_pinned_memory issues:
+	 *
+	 *     4) Is glMapBuffer on a shared buffer guaranteed to return the
+	 *        same system address which was specified at creation time?
+	 *
+	 *        RESOLVED: NO. The GL implementation might return a different
+	 *        virtual mapping of that memory, although the same physical
+	 *        page will be used.
+	 *
+	 * So don't ever use staging buffers.
+	 */
+	if (rbuffer->b.is_user_ptr)
+		usage |= PIPE_TRANSFER_PERSISTENT;
+
+	/* See if the buffer range being mapped has never been initialized,
+	 * in which case it can be mapped unsynchronized. */
+	if (!(usage & (PIPE_TRANSFER_UNSYNCHRONIZED |
+		       TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED)) &&
+	    usage & PIPE_TRANSFER_WRITE &&
+	    !rbuffer->b.is_shared &&
+	    !util_ranges_intersect(&rbuffer->valid_buffer_range, box->x, box->x + box->width)) {
+		usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
+	}
+
+	/* If discarding the entire range, discard the whole resource instead. */
+	if (usage & PIPE_TRANSFER_DISCARD_RANGE &&
+	    box->x == 0 && box->width == resource->width0) {
+		usage |= PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE;
+	}
+
+	if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE &&
+	    !(usage & (PIPE_TRANSFER_UNSYNCHRONIZED |
+		       TC_TRANSFER_MAP_NO_INVALIDATE))) {
+		assert(usage & PIPE_TRANSFER_WRITE);
+
+		if (r600_invalidate_buffer(rctx, rbuffer)) {
+			/* At this point, the buffer is always idle. */
+			usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
+		} else {
+			/* Fall back to a temporary buffer. */
+			usage |= PIPE_TRANSFER_DISCARD_RANGE;
+		}
+	}
+
+	if ((usage & PIPE_TRANSFER_DISCARD_RANGE) &&
+	    !(rscreen->debug_flags & DBG_NO_DISCARD_RANGE) &&
+	    ((!(usage & (PIPE_TRANSFER_UNSYNCHRONIZED |
+			 PIPE_TRANSFER_PERSISTENT)) &&
+	      r600_can_dma_copy_buffer(rctx, box->x, 0, box->width)) ||
+	     (rbuffer->flags & RADEON_FLAG_SPARSE))) {
+		assert(usage & PIPE_TRANSFER_WRITE);
+
+		/* Check if mapping this buffer would cause waiting for the GPU.
+		 */
+		if (rbuffer->flags & RADEON_FLAG_SPARSE ||
+		    r600_rings_is_buffer_referenced(rctx, rbuffer->buf, RADEON_USAGE_READWRITE) ||
+		    !rctx->ws->buffer_wait(rbuffer->buf, 0, RADEON_USAGE_READWRITE)) {
+			/* Do a wait-free write-only transfer using a temporary buffer. */
+			unsigned offset;
+			struct r600_resource *staging = NULL;
+
+			u_upload_alloc(ctx->stream_uploader, 0,
+                                       box->width + (box->x % R600_MAP_BUFFER_ALIGNMENT),
+				       rctx->screen->info.tcc_cache_line_size,
+				       &offset, (struct pipe_resource**)&staging,
+                                       (void**)&data);
+
+			if (staging) {
+				data += box->x % R600_MAP_BUFFER_ALIGNMENT;
+				return r600_buffer_get_transfer(ctx, resource, usage, box,
+								ptransfer, data, staging, offset);
+			} else if (rbuffer->flags & RADEON_FLAG_SPARSE) {
+				return NULL;
+			}
+		} else {
+			/* At this point, the buffer is always idle (we checked it above). */
+			usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
+		}
+	}
+	/* Use a staging buffer in cached GTT for reads. */
+	else if (((usage & PIPE_TRANSFER_READ) &&
+		  !(usage & PIPE_TRANSFER_PERSISTENT) &&
+		  (rbuffer->domains & RADEON_DOMAIN_VRAM ||
+		   rbuffer->flags & RADEON_FLAG_GTT_WC) &&
+		  r600_can_dma_copy_buffer(rctx, 0, box->x, box->width)) ||
+		 (rbuffer->flags & RADEON_FLAG_SPARSE)) {
+		struct r600_resource *staging;
+
+		assert(!(usage & TC_TRANSFER_MAP_THREADED_UNSYNC));
+		staging = (struct r600_resource*) pipe_buffer_create(
+				ctx->screen, 0, PIPE_USAGE_STAGING,
+				box->width + (box->x % R600_MAP_BUFFER_ALIGNMENT));
+		if (staging) {
+			/* Copy the VRAM buffer to the staging buffer. */
+			rctx->dma_copy(ctx, &staging->b.b, 0,
+				       box->x % R600_MAP_BUFFER_ALIGNMENT,
+				       0, 0, resource, 0, box);
+
+			data = r600_buffer_map_sync_with_rings(rctx, staging,
+							       usage & ~PIPE_TRANSFER_UNSYNCHRONIZED);
+			if (!data) {
+				r600_resource_reference(&staging, NULL);
+				return NULL;
+			}
+			data += box->x % R600_MAP_BUFFER_ALIGNMENT;
+
+			return r600_buffer_get_transfer(ctx, resource, usage, box,
+							ptransfer, data, staging, 0);
+		} else if (rbuffer->flags & RADEON_FLAG_SPARSE) {
+			return NULL;
+		}
+	}
+
+	data = r600_buffer_map_sync_with_rings(rctx, rbuffer, usage);
+	if (!data) {
+		return NULL;
+	}
+	data += box->x;
+
+	return r600_buffer_get_transfer(ctx, resource, usage, box,
+					ptransfer, data, NULL, 0);
+}
+
+static void r600_buffer_do_flush_region(struct pipe_context *ctx,
+					struct pipe_transfer *transfer,
+				        const struct pipe_box *box)
+{
+	struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
+	struct r600_resource *rbuffer = r600_resource(transfer->resource);
+
+	if (rtransfer->staging) {
+		struct pipe_resource *dst, *src;
+		unsigned soffset;
+		struct pipe_box dma_box;
+
+		dst = transfer->resource;
+		src = &rtransfer->staging->b.b;
+		soffset = rtransfer->offset + box->x % R600_MAP_BUFFER_ALIGNMENT;
+
+		u_box_1d(soffset, box->width, &dma_box);
+
+		/* Copy the staging buffer into the original one. */
+		ctx->resource_copy_region(ctx, dst, 0, box->x, 0, 0, src, 0, &dma_box);
+	}
+
+	util_range_add(&rbuffer->valid_buffer_range, box->x,
+		       box->x + box->width);
+}
+
+static void r600_buffer_flush_region(struct pipe_context *ctx,
+				     struct pipe_transfer *transfer,
+				     const struct pipe_box *rel_box)
+{
+	unsigned required_usage = PIPE_TRANSFER_WRITE |
+				  PIPE_TRANSFER_FLUSH_EXPLICIT;
+
+	if ((transfer->usage & required_usage) == required_usage) {
+		struct pipe_box box;
+
+		u_box_1d(transfer->box.x + rel_box->x, rel_box->width, &box);
+		r600_buffer_do_flush_region(ctx, transfer, &box);
+	}
+}
+
+static void r600_buffer_transfer_unmap(struct pipe_context *ctx,
+				       struct pipe_transfer *transfer)
+{
+	struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+	struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
+
+	if (transfer->usage & PIPE_TRANSFER_WRITE &&
+	    !(transfer->usage & PIPE_TRANSFER_FLUSH_EXPLICIT))
+		r600_buffer_do_flush_region(ctx, transfer, &transfer->box);
+
+	r600_resource_reference(&rtransfer->staging, NULL);
+	assert(rtransfer->b.staging == NULL); /* for threaded context only */
+	pipe_resource_reference(&transfer->resource, NULL);
+
+	/* Don't use pool_transfers_unsync. We are always in the driver
+	 * thread. */
+	slab_free(&rctx->pool_transfers, transfer);
+}
+
+void r600_buffer_subdata(struct pipe_context *ctx,
+			 struct pipe_resource *buffer,
+			 unsigned usage, unsigned offset,
+			 unsigned size, const void *data)
+{
+	struct pipe_transfer *transfer = NULL;
+	struct pipe_box box;
+	uint8_t *map = NULL;
+
+	u_box_1d(offset, size, &box);
+	map = r600_buffer_transfer_map(ctx, buffer, 0,
+				       PIPE_TRANSFER_WRITE |
+				       PIPE_TRANSFER_DISCARD_RANGE |
+				       usage,
+				       &box, &transfer);
+	if (!map)
+		return;
+
+	memcpy(map, data, size);
+	r600_buffer_transfer_unmap(ctx, transfer);
+}
+
+static const struct u_resource_vtbl r600_buffer_vtbl =
+{
+	NULL,				/* get_handle */
+	r600_buffer_destroy,		/* resource_destroy */
+	r600_buffer_transfer_map,	/* transfer_map */
+	r600_buffer_flush_region,	/* transfer_flush_region */
+	r600_buffer_transfer_unmap,	/* transfer_unmap */
+};
+
+static struct r600_resource *
+r600_alloc_buffer_struct(struct pipe_screen *screen,
+			 const struct pipe_resource *templ)
+{
+	struct r600_resource *rbuffer;
+
+	rbuffer = MALLOC_STRUCT(r600_resource);
+
+	rbuffer->b.b = *templ;
+	rbuffer->b.b.next = NULL;
+	pipe_reference_init(&rbuffer->b.b.reference, 1);
+	rbuffer->b.b.screen = screen;
+
+	rbuffer->b.vtbl = &r600_buffer_vtbl;
+	threaded_resource_init(&rbuffer->b.b);
+
+	rbuffer->buf = NULL;
+	rbuffer->bind_history = 0;
+	rbuffer->TC_L2_dirty = false;
+	util_range_init(&rbuffer->valid_buffer_range);
+	return rbuffer;
+}
+
+struct pipe_resource *r600_buffer_create(struct pipe_screen *screen,
+					 const struct pipe_resource *templ,
+					 unsigned alignment)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+	struct r600_resource *rbuffer = r600_alloc_buffer_struct(screen, templ);
+
+	r600_init_resource_fields(rscreen, rbuffer, templ->width0, alignment);
+
+	if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE)
+		rbuffer->flags |= RADEON_FLAG_SPARSE;
+
+	if (!r600_alloc_resource(rscreen, rbuffer)) {
+		FREE(rbuffer);
+		return NULL;
+	}
+	return &rbuffer->b.b;
+}
+
+struct pipe_resource *r600_aligned_buffer_create(struct pipe_screen *screen,
+						 unsigned flags,
+						 unsigned usage,
+						 unsigned size,
+						 unsigned alignment)
+{
+	struct pipe_resource buffer;
+
+	memset(&buffer, 0, sizeof buffer);
+	buffer.target = PIPE_BUFFER;
+	buffer.format = PIPE_FORMAT_R8_UNORM;
+	buffer.bind = 0;
+	buffer.usage = usage;
+	buffer.flags = flags;
+	buffer.width0 = size;
+	buffer.height0 = 1;
+	buffer.depth0 = 1;
+	buffer.array_size = 1;
+	return r600_buffer_create(screen, &buffer, alignment);
+}
+
+struct pipe_resource *
+r600_buffer_from_user_memory(struct pipe_screen *screen,
+			     const struct pipe_resource *templ,
+			     void *user_memory)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+	struct radeon_winsys *ws = rscreen->ws;
+	struct r600_resource *rbuffer = r600_alloc_buffer_struct(screen, templ);
+
+	rbuffer->domains = RADEON_DOMAIN_GTT;
+	rbuffer->flags = 0;
+	rbuffer->b.is_user_ptr = true;
+	util_range_add(&rbuffer->valid_buffer_range, 0, templ->width0);
+	util_range_add(&rbuffer->b.valid_buffer_range, 0, templ->width0);
+
+	/* Convert a user pointer to a buffer. */
+	rbuffer->buf = ws->buffer_from_ptr(ws, user_memory, templ->width0);
+	if (!rbuffer->buf) {
+		FREE(rbuffer);
+		return NULL;
+	}
+
+	if (rscreen->info.has_virtual_memory)
+		rbuffer->gpu_address =
+			ws->buffer_get_virtual_address(rbuffer->buf);
+	else
+		rbuffer->gpu_address = 0;
+
+	rbuffer->vram_usage = 0;
+	rbuffer->gart_usage = templ->width0;
+
+	return &rbuffer->b.b;
+}
diff --git a/src/gallium/drivers/r600/r600_cs.h b/src/gallium/drivers/r600/r600_cs.h
new file mode 100644
index 00000000000..28bdf15b8ad
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_cs.h
@@ -0,0 +1,209 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Marek Olšák <[email protected]>
+ */
+
+/**
+ * This file contains helpers for writing commands to commands streams.
+ */
+
+#ifndef R600_CS_H
+#define R600_CS_H
+
+#include "r600_pipe_common.h"
+#include "amd/common/r600d_common.h"
+
+/**
+ * Return true if there is enough memory in VRAM and GTT for the buffers
+ * added so far.
+ *
+ * \param vram      VRAM memory size not added to the buffer list yet
+ * \param gtt       GTT memory size not added to the buffer list yet
+ */
+static inline bool
+radeon_cs_memory_below_limit(struct r600_common_screen *screen,
+			     struct radeon_winsys_cs *cs,
+			     uint64_t vram, uint64_t gtt)
+{
+	vram += cs->used_vram;
+	gtt += cs->used_gart;
+
+	/* Anything that goes above the VRAM size should go to GTT. */
+	if (vram > screen->info.vram_size)
+		gtt += vram - screen->info.vram_size;
+
+	/* Now we just need to check if we have enough GTT. */
+	return gtt < screen->info.gart_size * 0.7;
+}
+
+/**
+ * Add a buffer to the buffer list for the given command stream (CS).
+ *
+ * All buffers used by a CS must be added to the list. This tells the kernel
+ * driver which buffers are used by GPU commands. Other buffers can
+ * be swapped out (not accessible) during execution.
+ *
+ * The buffer list becomes empty after every context flush and must be
+ * rebuilt.
+ */
+static inline unsigned radeon_add_to_buffer_list(struct r600_common_context *rctx,
+						 struct r600_ring *ring,
+						 struct r600_resource *rbo,
+						 enum radeon_bo_usage usage,
+						 enum radeon_bo_priority priority)
+{
+	assert(usage);
+	return rctx->ws->cs_add_buffer(
+		ring->cs, rbo->buf,
+		(enum radeon_bo_usage)(usage | RADEON_USAGE_SYNCHRONIZED),
+		rbo->domains, priority) * 4;
+}
+
+/**
+ * Same as above, but also checks memory usage and flushes the context
+ * accordingly.
+ *
+ * When this SHOULD NOT be used:
+ *
+ * - if r600_context_add_resource_size has been called for the buffer
+ *   followed by *_need_cs_space for checking the memory usage
+ *
+ * - if r600_need_dma_space has been called for the buffer
+ *
+ * - when emitting state packets and draw packets (because preceding packets
+ *   can't be re-emitted at that point)
+ *
+ * - if shader resource "enabled_mask" is not up-to-date or there is
+ *   a different constraint disallowing a context flush
+ */
+static inline unsigned
+radeon_add_to_buffer_list_check_mem(struct r600_common_context *rctx,
+				    struct r600_ring *ring,
+				    struct r600_resource *rbo,
+				    enum radeon_bo_usage usage,
+				    enum radeon_bo_priority priority,
+				    bool check_mem)
+{
+	if (check_mem &&
+	    !radeon_cs_memory_below_limit(rctx->screen, ring->cs,
+					  rctx->vram + rbo->vram_usage,
+					  rctx->gtt + rbo->gart_usage))
+		ring->flush(rctx, RADEON_FLUSH_ASYNC, NULL);
+
+	return radeon_add_to_buffer_list(rctx, ring, rbo, usage, priority);
+}
+
+static inline void r600_emit_reloc(struct r600_common_context *rctx,
+				   struct r600_ring *ring, struct r600_resource *rbo,
+				   enum radeon_bo_usage usage,
+				   enum radeon_bo_priority priority)
+{
+	struct radeon_winsys_cs *cs = ring->cs;
+	bool has_vm = ((struct r600_common_screen*)rctx->b.screen)->info.has_virtual_memory;
+	unsigned reloc = radeon_add_to_buffer_list(rctx, ring, rbo, usage, priority);
+
+	if (!has_vm) {
+		radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
+		radeon_emit(cs, reloc);
+	}
+}
+
+static inline void radeon_set_config_reg_seq(struct radeon_winsys_cs *cs, unsigned reg, unsigned num)
+{
+	assert(reg < R600_CONTEXT_REG_OFFSET);
+	assert(cs->current.cdw + 2 + num <= cs->current.max_dw);
+	radeon_emit(cs, PKT3(PKT3_SET_CONFIG_REG, num, 0));
+	radeon_emit(cs, (reg - R600_CONFIG_REG_OFFSET) >> 2);
+}
+
+static inline void radeon_set_config_reg(struct radeon_winsys_cs *cs, unsigned reg, unsigned value)
+{
+	radeon_set_config_reg_seq(cs, reg, 1);
+	radeon_emit(cs, value);
+}
+
+static inline void radeon_set_context_reg_seq(struct radeon_winsys_cs *cs, unsigned reg, unsigned num)
+{
+	assert(reg >= R600_CONTEXT_REG_OFFSET);
+	assert(cs->current.cdw + 2 + num <= cs->current.max_dw);
+	radeon_emit(cs, PKT3(PKT3_SET_CONTEXT_REG, num, 0));
+	radeon_emit(cs, (reg - R600_CONTEXT_REG_OFFSET) >> 2);
+}
+
+static inline void radeon_set_context_reg(struct radeon_winsys_cs *cs, unsigned reg, unsigned value)
+{
+	radeon_set_context_reg_seq(cs, reg, 1);
+	radeon_emit(cs, value);
+}
+
+static inline void radeon_set_context_reg_idx(struct radeon_winsys_cs *cs,
+					      unsigned reg, unsigned idx,
+					      unsigned value)
+{
+	assert(reg >= R600_CONTEXT_REG_OFFSET);
+	assert(cs->current.cdw + 3 <= cs->current.max_dw);
+	radeon_emit(cs, PKT3(PKT3_SET_CONTEXT_REG, 1, 0));
+	radeon_emit(cs, (reg - R600_CONTEXT_REG_OFFSET) >> 2 | (idx << 28));
+	radeon_emit(cs, value);
+}
+
+static inline void radeon_set_sh_reg_seq(struct radeon_winsys_cs *cs, unsigned reg, unsigned num)
+{
+	assert(reg >= SI_SH_REG_OFFSET && reg < SI_SH_REG_END);
+	assert(cs->current.cdw + 2 + num <= cs->current.max_dw);
+	radeon_emit(cs, PKT3(PKT3_SET_SH_REG, num, 0));
+	radeon_emit(cs, (reg - SI_SH_REG_OFFSET) >> 2);
+}
+
+static inline void radeon_set_sh_reg(struct radeon_winsys_cs *cs, unsigned reg, unsigned value)
+{
+	radeon_set_sh_reg_seq(cs, reg, 1);
+	radeon_emit(cs, value);
+}
+
+static inline void radeon_set_uconfig_reg_seq(struct radeon_winsys_cs *cs, unsigned reg, unsigned num)
+{
+	assert(reg >= CIK_UCONFIG_REG_OFFSET && reg < CIK_UCONFIG_REG_END);
+	assert(cs->current.cdw + 2 + num <= cs->current.max_dw);
+	radeon_emit(cs, PKT3(PKT3_SET_UCONFIG_REG, num, 0));
+	radeon_emit(cs, (reg - CIK_UCONFIG_REG_OFFSET) >> 2);
+}
+
+static inline void radeon_set_uconfig_reg(struct radeon_winsys_cs *cs, unsigned reg, unsigned value)
+{
+	radeon_set_uconfig_reg_seq(cs, reg, 1);
+	radeon_emit(cs, value);
+}
+
+static inline void radeon_set_uconfig_reg_idx(struct radeon_winsys_cs *cs,
+					      unsigned reg, unsigned idx,
+					      unsigned value)
+{
+	assert(reg >= CIK_UCONFIG_REG_OFFSET && reg < CIK_UCONFIG_REG_END);
+	assert(cs->current.cdw + 3 <= cs->current.max_dw);
+	radeon_emit(cs, PKT3(PKT3_SET_UCONFIG_REG, 1, 0));
+	radeon_emit(cs, (reg - CIK_UCONFIG_REG_OFFSET) >> 2 | (idx << 28));
+	radeon_emit(cs, value);
+}
+
+#endif
diff --git a/src/gallium/drivers/r600/r600_gpu_load.c b/src/gallium/drivers/r600/r600_gpu_load.c
new file mode 100644
index 00000000000..d35be4f327a
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_gpu_load.c
@@ -0,0 +1,283 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors: Marek Olšák <[email protected]>
+ *
+ */
+
+/* The GPU load is measured as follows.
+ *
+ * There is a thread which samples the GRBM_STATUS register at a certain
+ * frequency and the "busy" or "idle" counter is incremented based on
+ * whether the GUI_ACTIVE bit is set or not.
+ *
+ * Then, the user can sample the counters twice and calculate the average
+ * GPU load between the two samples.
+ */
+
+#include "r600_pipe_common.h"
+#include "r600_query.h"
+#include "os/os_time.h"
+
+/* For good accuracy at 1000 fps or lower. This will be inaccurate for higher
+ * fps (there are too few samples per frame). */
+#define SAMPLES_PER_SEC 10000
+
+#define GRBM_STATUS		0x8010
+#define TA_BUSY(x)		(((x) >> 14) & 0x1)
+#define GDS_BUSY(x)		(((x) >> 15) & 0x1)
+#define VGT_BUSY(x)		(((x) >> 17) & 0x1)
+#define IA_BUSY(x)		(((x) >> 19) & 0x1)
+#define SX_BUSY(x)		(((x) >> 20) & 0x1)
+#define WD_BUSY(x)		(((x) >> 21) & 0x1)
+#define SPI_BUSY(x)		(((x) >> 22) & 0x1)
+#define BCI_BUSY(x)		(((x) >> 23) & 0x1)
+#define SC_BUSY(x)		(((x) >> 24) & 0x1)
+#define PA_BUSY(x)		(((x) >> 25) & 0x1)
+#define DB_BUSY(x)		(((x) >> 26) & 0x1)
+#define CP_BUSY(x)		(((x) >> 29) & 0x1)
+#define CB_BUSY(x)		(((x) >> 30) & 0x1)
+#define GUI_ACTIVE(x)		(((x) >> 31) & 0x1)
+
+#define SRBM_STATUS2		0x0e4c
+#define SDMA_BUSY(x)		(((x) >> 5) & 0x1)
+
+#define CP_STAT                 0x8680
+#define PFP_BUSY(x)		(((x) >> 15) & 0x1)
+#define MEQ_BUSY(x)		(((x) >> 16) & 0x1)
+#define ME_BUSY(x)		(((x) >> 17) & 0x1)
+#define SURFACE_SYNC_BUSY(x)	(((x) >> 21) & 0x1)
+#define DMA_BUSY(x)		(((x) >> 22) & 0x1)
+#define SCRATCH_RAM_BUSY(x)	(((x) >> 24) & 0x1)
+
+#define IDENTITY(x) x
+
+#define UPDATE_COUNTER(field, mask)					\
+	do {								\
+		if (mask(value))					\
+			p_atomic_inc(&counters->named.field.busy);	\
+		else							\
+			p_atomic_inc(&counters->named.field.idle);	\
+	} while (0)
+
+static void r600_update_mmio_counters(struct r600_common_screen *rscreen,
+				      union r600_mmio_counters *counters)
+{
+	uint32_t value = 0;
+	bool gui_busy, sdma_busy = false;
+
+	/* GRBM_STATUS */
+	rscreen->ws->read_registers(rscreen->ws, GRBM_STATUS, 1, &value);
+
+	UPDATE_COUNTER(ta, TA_BUSY);
+	UPDATE_COUNTER(gds, GDS_BUSY);
+	UPDATE_COUNTER(vgt, VGT_BUSY);
+	UPDATE_COUNTER(ia, IA_BUSY);
+	UPDATE_COUNTER(sx, SX_BUSY);
+	UPDATE_COUNTER(wd, WD_BUSY);
+	UPDATE_COUNTER(spi, SPI_BUSY);
+	UPDATE_COUNTER(bci, BCI_BUSY);
+	UPDATE_COUNTER(sc, SC_BUSY);
+	UPDATE_COUNTER(pa, PA_BUSY);
+	UPDATE_COUNTER(db, DB_BUSY);
+	UPDATE_COUNTER(cp, CP_BUSY);
+	UPDATE_COUNTER(cb, CB_BUSY);
+	UPDATE_COUNTER(gui, GUI_ACTIVE);
+	gui_busy = GUI_ACTIVE(value);
+
+	if (rscreen->chip_class == CIK || rscreen->chip_class == VI) {
+		/* SRBM_STATUS2 */
+		rscreen->ws->read_registers(rscreen->ws, SRBM_STATUS2, 1, &value);
+
+		UPDATE_COUNTER(sdma, SDMA_BUSY);
+		sdma_busy = SDMA_BUSY(value);
+	}
+
+	if (rscreen->chip_class >= VI) {
+		/* CP_STAT */
+		rscreen->ws->read_registers(rscreen->ws, CP_STAT, 1, &value);
+
+		UPDATE_COUNTER(pfp, PFP_BUSY);
+		UPDATE_COUNTER(meq, MEQ_BUSY);
+		UPDATE_COUNTER(me, ME_BUSY);
+		UPDATE_COUNTER(surf_sync, SURFACE_SYNC_BUSY);
+		UPDATE_COUNTER(cp_dma, DMA_BUSY);
+		UPDATE_COUNTER(scratch_ram, SCRATCH_RAM_BUSY);
+	}
+
+	value = gui_busy || sdma_busy;
+	UPDATE_COUNTER(gpu, IDENTITY);
+}
+
+#undef UPDATE_COUNTER
+
+static int
+r600_gpu_load_thread(void *param)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)param;
+	const int period_us = 1000000 / SAMPLES_PER_SEC;
+	int sleep_us = period_us;
+	int64_t cur_time, last_time = os_time_get();
+
+	while (!p_atomic_read(&rscreen->gpu_load_stop_thread)) {
+		if (sleep_us)
+			os_time_sleep(sleep_us);
+
+		/* Make sure we sleep the ideal amount of time to match
+		 * the expected frequency. */
+		cur_time = os_time_get();
+
+		if (os_time_timeout(last_time, last_time + period_us,
+				    cur_time))
+			sleep_us = MAX2(sleep_us - 1, 1);
+		else
+			sleep_us += 1;
+
+		/*printf("Hz: %.1f\n", 1000000.0 / (cur_time - last_time));*/
+		last_time = cur_time;
+
+		/* Update the counters. */
+		r600_update_mmio_counters(rscreen, &rscreen->mmio_counters);
+	}
+	p_atomic_dec(&rscreen->gpu_load_stop_thread);
+	return 0;
+}
+
+void r600_gpu_load_kill_thread(struct r600_common_screen *rscreen)
+{
+	if (!rscreen->gpu_load_thread)
+		return;
+
+	p_atomic_inc(&rscreen->gpu_load_stop_thread);
+	thrd_join(rscreen->gpu_load_thread, NULL);
+	rscreen->gpu_load_thread = 0;
+}
+
+static uint64_t r600_read_mmio_counter(struct r600_common_screen *rscreen,
+				       unsigned busy_index)
+{
+	/* Start the thread if needed. */
+	if (!rscreen->gpu_load_thread) {
+		mtx_lock(&rscreen->gpu_load_mutex);
+		/* Check again inside the mutex. */
+		if (!rscreen->gpu_load_thread)
+			rscreen->gpu_load_thread =
+				u_thread_create(r600_gpu_load_thread, rscreen);
+		mtx_unlock(&rscreen->gpu_load_mutex);
+	}
+
+	unsigned busy = p_atomic_read(&rscreen->mmio_counters.array[busy_index]);
+	unsigned idle = p_atomic_read(&rscreen->mmio_counters.array[busy_index + 1]);
+
+	return busy | ((uint64_t)idle << 32);
+}
+
+static unsigned r600_end_mmio_counter(struct r600_common_screen *rscreen,
+				      uint64_t begin, unsigned busy_index)
+{
+	uint64_t end = r600_read_mmio_counter(rscreen, busy_index);
+	unsigned busy = (end & 0xffffffff) - (begin & 0xffffffff);
+	unsigned idle = (end >> 32) - (begin >> 32);
+
+	/* Calculate the % of time the busy counter was being incremented.
+	 *
+	 * If no counters were incremented, return the current counter status.
+	 * It's for the case when the load is queried faster than
+	 * the counters are updated.
+	 */
+	if (idle || busy) {
+		return busy*100 / (busy + idle);
+	} else {
+		union r600_mmio_counters counters;
+
+		memset(&counters, 0, sizeof(counters));
+		r600_update_mmio_counters(rscreen, &counters);
+		return counters.array[busy_index] ? 100 : 0;
+	}
+}
+
+#define BUSY_INDEX(rscreen, field) (&rscreen->mmio_counters.named.field.busy - \
+				    rscreen->mmio_counters.array)
+
+static unsigned busy_index_from_type(struct r600_common_screen *rscreen,
+				     unsigned type)
+{
+	switch (type) {
+	case R600_QUERY_GPU_LOAD:
+		return BUSY_INDEX(rscreen, gpu);
+	case R600_QUERY_GPU_SHADERS_BUSY:
+		return BUSY_INDEX(rscreen, spi);
+	case R600_QUERY_GPU_TA_BUSY:
+		return BUSY_INDEX(rscreen, ta);
+	case R600_QUERY_GPU_GDS_BUSY:
+		return BUSY_INDEX(rscreen, gds);
+	case R600_QUERY_GPU_VGT_BUSY:
+		return BUSY_INDEX(rscreen, vgt);
+	case R600_QUERY_GPU_IA_BUSY:
+		return BUSY_INDEX(rscreen, ia);
+	case R600_QUERY_GPU_SX_BUSY:
+		return BUSY_INDEX(rscreen, sx);
+	case R600_QUERY_GPU_WD_BUSY:
+		return BUSY_INDEX(rscreen, wd);
+	case R600_QUERY_GPU_BCI_BUSY:
+		return BUSY_INDEX(rscreen, bci);
+	case R600_QUERY_GPU_SC_BUSY:
+		return BUSY_INDEX(rscreen, sc);
+	case R600_QUERY_GPU_PA_BUSY:
+		return BUSY_INDEX(rscreen, pa);
+	case R600_QUERY_GPU_DB_BUSY:
+		return BUSY_INDEX(rscreen, db);
+	case R600_QUERY_GPU_CP_BUSY:
+		return BUSY_INDEX(rscreen, cp);
+	case R600_QUERY_GPU_CB_BUSY:
+		return BUSY_INDEX(rscreen, cb);
+	case R600_QUERY_GPU_SDMA_BUSY:
+		return BUSY_INDEX(rscreen, sdma);
+	case R600_QUERY_GPU_PFP_BUSY:
+		return BUSY_INDEX(rscreen, pfp);
+	case R600_QUERY_GPU_MEQ_BUSY:
+		return BUSY_INDEX(rscreen, meq);
+	case R600_QUERY_GPU_ME_BUSY:
+		return BUSY_INDEX(rscreen, me);
+	case R600_QUERY_GPU_SURF_SYNC_BUSY:
+		return BUSY_INDEX(rscreen, surf_sync);
+	case R600_QUERY_GPU_CP_DMA_BUSY:
+		return BUSY_INDEX(rscreen, cp_dma);
+	case R600_QUERY_GPU_SCRATCH_RAM_BUSY:
+		return BUSY_INDEX(rscreen, scratch_ram);
+	default:
+		unreachable("invalid query type");
+	}
+}
+
+uint64_t r600_begin_counter(struct r600_common_screen *rscreen, unsigned type)
+{
+	unsigned busy_index = busy_index_from_type(rscreen, type);
+	return r600_read_mmio_counter(rscreen, busy_index);
+}
+
+unsigned r600_end_counter(struct r600_common_screen *rscreen, unsigned type,
+			  uint64_t begin)
+{
+	unsigned busy_index = busy_index_from_type(rscreen, type);
+	return r600_end_mmio_counter(rscreen, begin, busy_index);
+}
diff --git a/src/gallium/drivers/r600/r600_perfcounter.c b/src/gallium/drivers/r600/r600_perfcounter.c
new file mode 100644
index 00000000000..48f609bcb41
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_perfcounter.c
@@ -0,0 +1,649 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *  Nicolai Hähnle <[email protected]>
+ *
+ */
+
+#include "util/u_memory.h"
+#include "r600_query.h"
+#include "r600_pipe_common.h"
+#include "amd/common/r600d_common.h"
+
+/* Max counters per HW block */
+#define R600_QUERY_MAX_COUNTERS 16
+
+static struct r600_perfcounter_block *
+lookup_counter(struct r600_perfcounters *pc, unsigned index,
+	       unsigned *base_gid, unsigned *sub_index)
+{
+	struct r600_perfcounter_block *block = pc->blocks;
+	unsigned bid;
+
+	*base_gid = 0;
+	for (bid = 0; bid < pc->num_blocks; ++bid, ++block) {
+		unsigned total = block->num_groups * block->num_selectors;
+
+		if (index < total) {
+			*sub_index = index;
+			return block;
+		}
+
+		index -= total;
+		*base_gid += block->num_groups;
+	}
+
+	return NULL;
+}
+
+static struct r600_perfcounter_block *
+lookup_group(struct r600_perfcounters *pc, unsigned *index)
+{
+	unsigned bid;
+	struct r600_perfcounter_block *block = pc->blocks;
+
+	for (bid = 0; bid < pc->num_blocks; ++bid, ++block) {
+		if (*index < block->num_groups)
+			return block;
+		*index -= block->num_groups;
+	}
+
+	return NULL;
+}
+
+struct r600_pc_group {
+	struct r600_pc_group *next;
+	struct r600_perfcounter_block *block;
+	unsigned sub_gid; /* only used during init */
+	unsigned result_base; /* only used during init */
+	int se;
+	int instance;
+	unsigned num_counters;
+	unsigned selectors[R600_QUERY_MAX_COUNTERS];
+};
+
+struct r600_pc_counter {
+	unsigned base;
+	unsigned qwords;
+	unsigned stride; /* in uint64s */
+};
+
+#define R600_PC_SHADERS_WINDOWING (1 << 31)
+
+struct r600_query_pc {
+	struct r600_query_hw b;
+
+	unsigned shaders;
+	unsigned num_counters;
+	struct r600_pc_counter *counters;
+	struct r600_pc_group *groups;
+};
+
+static void r600_pc_query_destroy(struct r600_common_screen *rscreen,
+				  struct r600_query *rquery)
+{
+	struct r600_query_pc *query = (struct r600_query_pc *)rquery;
+
+	while (query->groups) {
+		struct r600_pc_group *group = query->groups;
+		query->groups = group->next;
+		FREE(group);
+	}
+
+	FREE(query->counters);
+
+	r600_query_hw_destroy(rscreen, rquery);
+}
+
+static bool r600_pc_query_prepare_buffer(struct r600_common_screen *screen,
+					 struct r600_query_hw *hwquery,
+					 struct r600_resource *buffer)
+{
+	/* no-op */
+	return true;
+}
+
+static void r600_pc_query_emit_start(struct r600_common_context *ctx,
+				     struct r600_query_hw *hwquery,
+				     struct r600_resource *buffer, uint64_t va)
+{
+	struct r600_perfcounters *pc = ctx->screen->perfcounters;
+	struct r600_query_pc *query = (struct r600_query_pc *)hwquery;
+	struct r600_pc_group *group;
+	int current_se = -1;
+	int current_instance = -1;
+
+	if (query->shaders)
+		pc->emit_shaders(ctx, query->shaders);
+
+	for (group = query->groups; group; group = group->next) {
+		struct r600_perfcounter_block *block = group->block;
+
+		if (group->se != current_se || group->instance != current_instance) {
+			current_se = group->se;
+			current_instance = group->instance;
+			pc->emit_instance(ctx, group->se, group->instance);
+		}
+
+		pc->emit_select(ctx, block, group->num_counters, group->selectors);
+	}
+
+	if (current_se != -1 || current_instance != -1)
+		pc->emit_instance(ctx, -1, -1);
+
+	pc->emit_start(ctx, buffer, va);
+}
+
+static void r600_pc_query_emit_stop(struct r600_common_context *ctx,
+				    struct r600_query_hw *hwquery,
+				    struct r600_resource *buffer, uint64_t va)
+{
+	struct r600_perfcounters *pc = ctx->screen->perfcounters;
+	struct r600_query_pc *query = (struct r600_query_pc *)hwquery;
+	struct r600_pc_group *group;
+
+	pc->emit_stop(ctx, buffer, va);
+
+	for (group = query->groups; group; group = group->next) {
+		struct r600_perfcounter_block *block = group->block;
+		unsigned se = group->se >= 0 ? group->se : 0;
+		unsigned se_end = se + 1;
+
+		if ((block->flags & R600_PC_BLOCK_SE) && (group->se < 0))
+			se_end = ctx->screen->info.max_se;
+
+		do {
+			unsigned instance = group->instance >= 0 ? group->instance : 0;
+
+			do {
+				pc->emit_instance(ctx, se, instance);
+				pc->emit_read(ctx, block,
+					      group->num_counters, group->selectors,
+					      buffer, va);
+				va += sizeof(uint64_t) * group->num_counters;
+			} while (group->instance < 0 && ++instance < block->num_instances);
+		} while (++se < se_end);
+	}
+
+	pc->emit_instance(ctx, -1, -1);
+}
+
+static void r600_pc_query_clear_result(struct r600_query_hw *hwquery,
+				       union pipe_query_result *result)
+{
+	struct r600_query_pc *query = (struct r600_query_pc *)hwquery;
+
+	memset(result, 0, sizeof(result->batch[0]) * query->num_counters);
+}
+
+static void r600_pc_query_add_result(struct r600_common_screen *rscreen,
+				     struct r600_query_hw *hwquery,
+				     void *buffer,
+				     union pipe_query_result *result)
+{
+	struct r600_query_pc *query = (struct r600_query_pc *)hwquery;
+	uint64_t *results = buffer;
+	unsigned i, j;
+
+	for (i = 0; i < query->num_counters; ++i) {
+		struct r600_pc_counter *counter = &query->counters[i];
+
+		for (j = 0; j < counter->qwords; ++j) {
+			uint32_t value = results[counter->base + j * counter->stride];
+			result->batch[i].u64 += value;
+		}
+	}
+}
+
+static struct r600_query_ops batch_query_ops = {
+	.destroy = r600_pc_query_destroy,
+	.begin = r600_query_hw_begin,
+	.end = r600_query_hw_end,
+	.get_result = r600_query_hw_get_result
+};
+
+static struct r600_query_hw_ops batch_query_hw_ops = {
+	.prepare_buffer = r600_pc_query_prepare_buffer,
+	.emit_start = r600_pc_query_emit_start,
+	.emit_stop = r600_pc_query_emit_stop,
+	.clear_result = r600_pc_query_clear_result,
+	.add_result = r600_pc_query_add_result,
+};
+
+static struct r600_pc_group *get_group_state(struct r600_common_screen *screen,
+					     struct r600_query_pc *query,
+					     struct r600_perfcounter_block *block,
+					     unsigned sub_gid)
+{
+	struct r600_pc_group *group = query->groups;
+
+	while (group) {
+		if (group->block == block && group->sub_gid == sub_gid)
+			return group;
+		group = group->next;
+	}
+
+	group = CALLOC_STRUCT(r600_pc_group);
+	if (!group)
+		return NULL;
+
+	group->block = block;
+	group->sub_gid = sub_gid;
+
+	if (block->flags & R600_PC_BLOCK_SHADER) {
+		unsigned sub_gids = block->num_instances;
+		unsigned shader_id;
+		unsigned shaders;
+		unsigned query_shaders;
+
+		if (block->flags & R600_PC_BLOCK_SE_GROUPS)
+			sub_gids = sub_gids * screen->info.max_se;
+		shader_id = sub_gid / sub_gids;
+		sub_gid = sub_gid % sub_gids;
+
+		shaders = screen->perfcounters->shader_type_bits[shader_id];
+
+		query_shaders = query->shaders & ~R600_PC_SHADERS_WINDOWING;
+		if (query_shaders && query_shaders != shaders) {
+			fprintf(stderr, "r600_perfcounter: incompatible shader groups\n");
+			FREE(group);
+			return NULL;
+		}
+		query->shaders = shaders;
+	}
+
+	if (block->flags & R600_PC_BLOCK_SHADER_WINDOWED && !query->shaders) {
+		// A non-zero value in query->shaders ensures that the shader
+		// masking is reset unless the user explicitly requests one.
+		query->shaders = R600_PC_SHADERS_WINDOWING;
+	}
+
+	if (block->flags & R600_PC_BLOCK_SE_GROUPS) {
+		group->se = sub_gid / block->num_instances;
+		sub_gid = sub_gid % block->num_instances;
+	} else {
+		group->se = -1;
+	}
+
+	if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS) {
+		group->instance = sub_gid;
+	} else {
+		group->instance = -1;
+	}
+
+	group->next = query->groups;
+	query->groups = group;
+
+	return group;
+}
+
+struct pipe_query *r600_create_batch_query(struct pipe_context *ctx,
+					   unsigned num_queries,
+					   unsigned *query_types)
+{
+	struct r600_common_screen *screen =
+		(struct r600_common_screen *)ctx->screen;
+	struct r600_perfcounters *pc = screen->perfcounters;
+	struct r600_perfcounter_block *block;
+	struct r600_pc_group *group;
+	struct r600_query_pc *query;
+	unsigned base_gid, sub_gid, sub_index;
+	unsigned i, j;
+
+	if (!pc)
+		return NULL;
+
+	query = CALLOC_STRUCT(r600_query_pc);
+	if (!query)
+		return NULL;
+
+	query->b.b.ops = &batch_query_ops;
+	query->b.ops = &batch_query_hw_ops;
+
+	query->num_counters = num_queries;
+
+	/* Collect selectors per group */
+	for (i = 0; i < num_queries; ++i) {
+		unsigned sub_gid;
+
+		if (query_types[i] < R600_QUERY_FIRST_PERFCOUNTER)
+			goto error;
+
+		block = lookup_counter(pc, query_types[i] - R600_QUERY_FIRST_PERFCOUNTER,
+				       &base_gid, &sub_index);
+		if (!block)
+			goto error;
+
+		sub_gid = sub_index / block->num_selectors;
+		sub_index = sub_index % block->num_selectors;
+
+		group = get_group_state(screen, query, block, sub_gid);
+		if (!group)
+			goto error;
+
+		if (group->num_counters >= block->num_counters) {
+			fprintf(stderr,
+				"perfcounter group %s: too many selected\n",
+				block->basename);
+			goto error;
+		}
+		group->selectors[group->num_counters] = sub_index;
+		++group->num_counters;
+	}
+
+	/* Compute result bases and CS size per group */
+	query->b.num_cs_dw_begin = pc->num_start_cs_dwords;
+	query->b.num_cs_dw_end = pc->num_stop_cs_dwords;
+
+	query->b.num_cs_dw_begin += pc->num_instance_cs_dwords; /* conservative */
+	query->b.num_cs_dw_end += pc->num_instance_cs_dwords;
+
+	i = 0;
+	for (group = query->groups; group; group = group->next) {
+		struct r600_perfcounter_block *block = group->block;
+		unsigned select_dw, read_dw;
+		unsigned instances = 1;
+
+		if ((block->flags & R600_PC_BLOCK_SE) && group->se < 0)
+			instances = screen->info.max_se;
+		if (group->instance < 0)
+			instances *= block->num_instances;
+
+		group->result_base = i;
+		query->b.result_size += sizeof(uint64_t) * instances * group->num_counters;
+		i += instances * group->num_counters;
+
+		pc->get_size(block, group->num_counters, group->selectors,
+			     &select_dw, &read_dw);
+		query->b.num_cs_dw_begin += select_dw;
+		query->b.num_cs_dw_end += instances * read_dw;
+		query->b.num_cs_dw_begin += pc->num_instance_cs_dwords; /* conservative */
+		query->b.num_cs_dw_end += instances * pc->num_instance_cs_dwords;
+	}
+
+	if (query->shaders) {
+		if (query->shaders == R600_PC_SHADERS_WINDOWING)
+			query->shaders = 0xffffffff;
+		query->b.num_cs_dw_begin += pc->num_shaders_cs_dwords;
+	}
+
+	/* Map user-supplied query array to result indices */
+	query->counters = CALLOC(num_queries, sizeof(*query->counters));
+	for (i = 0; i < num_queries; ++i) {
+		struct r600_pc_counter *counter = &query->counters[i];
+		struct r600_perfcounter_block *block;
+
+		block = lookup_counter(pc, query_types[i] - R600_QUERY_FIRST_PERFCOUNTER,
+				       &base_gid, &sub_index);
+
+		sub_gid = sub_index / block->num_selectors;
+		sub_index = sub_index % block->num_selectors;
+
+		group = get_group_state(screen, query, block, sub_gid);
+		assert(group != NULL);
+
+		for (j = 0; j < group->num_counters; ++j) {
+			if (group->selectors[j] == sub_index)
+				break;
+		}
+
+		counter->base = group->result_base + j;
+		counter->stride = group->num_counters;
+
+		counter->qwords = 1;
+		if ((block->flags & R600_PC_BLOCK_SE) && group->se < 0)
+			counter->qwords = screen->info.max_se;
+		if (group->instance < 0)
+			counter->qwords *= block->num_instances;
+	}
+
+	if (!r600_query_hw_init(screen, &query->b))
+		goto error;
+
+	return (struct pipe_query *)query;
+
+error:
+	r600_pc_query_destroy(screen, &query->b.b);
+	return NULL;
+}
+
+static bool r600_init_block_names(struct r600_common_screen *screen,
+				  struct r600_perfcounter_block *block)
+{
+	unsigned i, j, k;
+	unsigned groups_shader = 1, groups_se = 1, groups_instance = 1;
+	unsigned namelen;
+	char *groupname;
+	char *p;
+
+	if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS)
+		groups_instance = block->num_instances;
+	if (block->flags & R600_PC_BLOCK_SE_GROUPS)
+		groups_se = screen->info.max_se;
+	if (block->flags & R600_PC_BLOCK_SHADER)
+		groups_shader = screen->perfcounters->num_shader_types;
+
+	namelen = strlen(block->basename);
+	block->group_name_stride = namelen + 1;
+	if (block->flags & R600_PC_BLOCK_SHADER)
+		block->group_name_stride += 3;
+	if (block->flags & R600_PC_BLOCK_SE_GROUPS) {
+		assert(groups_se <= 10);
+		block->group_name_stride += 1;
+
+		if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS)
+			block->group_name_stride += 1;
+	}
+	if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS) {
+		assert(groups_instance <= 100);
+		block->group_name_stride += 2;
+	}
+
+	block->group_names = MALLOC(block->num_groups * block->group_name_stride);
+	if (!block->group_names)
+		return false;
+
+	groupname = block->group_names;
+	for (i = 0; i < groups_shader; ++i) {
+		const char *shader_suffix = screen->perfcounters->shader_type_suffixes[i];
+		unsigned shaderlen = strlen(shader_suffix);
+		for (j = 0; j < groups_se; ++j) {
+			for (k = 0; k < groups_instance; ++k) {
+				strcpy(groupname, block->basename);
+				p = groupname + namelen;
+
+				if (block->flags & R600_PC_BLOCK_SHADER) {
+					strcpy(p, shader_suffix);
+					p += shaderlen;
+				}
+
+				if (block->flags & R600_PC_BLOCK_SE_GROUPS) {
+					p += sprintf(p, "%d", j);
+					if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS)
+						*p++ = '_';
+				}
+
+				if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS)
+					p += sprintf(p, "%d", k);
+
+				groupname += block->group_name_stride;
+			}
+		}
+	}
+
+	assert(block->num_selectors <= 1000);
+	block->selector_name_stride = block->group_name_stride + 4;
+	block->selector_names = MALLOC(block->num_groups * block->num_selectors *
+				       block->selector_name_stride);
+	if (!block->selector_names)
+		return false;
+
+	groupname = block->group_names;
+	p = block->selector_names;
+	for (i = 0; i < block->num_groups; ++i) {
+		for (j = 0; j < block->num_selectors; ++j) {
+			sprintf(p, "%s_%03d", groupname, j);
+			p += block->selector_name_stride;
+		}
+		groupname += block->group_name_stride;
+	}
+
+	return true;
+}
+
+int r600_get_perfcounter_info(struct r600_common_screen *screen,
+			      unsigned index,
+			      struct pipe_driver_query_info *info)
+{
+	struct r600_perfcounters *pc = screen->perfcounters;
+	struct r600_perfcounter_block *block;
+	unsigned base_gid, sub;
+
+	if (!pc)
+		return 0;
+
+	if (!info) {
+		unsigned bid, num_queries = 0;
+
+		for (bid = 0; bid < pc->num_blocks; ++bid) {
+			num_queries += pc->blocks[bid].num_selectors *
+				       pc->blocks[bid].num_groups;
+		}
+
+		return num_queries;
+	}
+
+	block = lookup_counter(pc, index, &base_gid, &sub);
+	if (!block)
+		return 0;
+
+	if (!block->selector_names) {
+		if (!r600_init_block_names(screen, block))
+			return 0;
+	}
+	info->name = block->selector_names + sub * block->selector_name_stride;
+	info->query_type = R600_QUERY_FIRST_PERFCOUNTER + index;
+	info->max_value.u64 = 0;
+	info->type = PIPE_DRIVER_QUERY_TYPE_UINT64;
+	info->result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_AVERAGE;
+	info->group_id = base_gid + sub / block->num_selectors;
+	info->flags = PIPE_DRIVER_QUERY_FLAG_BATCH;
+	if (sub > 0 && sub + 1 < block->num_selectors * block->num_groups)
+		info->flags |= PIPE_DRIVER_QUERY_FLAG_DONT_LIST;
+	return 1;
+}
+
+int r600_get_perfcounter_group_info(struct r600_common_screen *screen,
+				    unsigned index,
+				    struct pipe_driver_query_group_info *info)
+{
+	struct r600_perfcounters *pc = screen->perfcounters;
+	struct r600_perfcounter_block *block;
+
+	if (!pc)
+		return 0;
+
+	if (!info)
+		return pc->num_groups;
+
+	block = lookup_group(pc, &index);
+	if (!block)
+		return 0;
+
+	if (!block->group_names) {
+		if (!r600_init_block_names(screen, block))
+			return 0;
+	}
+	info->name = block->group_names + index * block->group_name_stride;
+	info->num_queries = block->num_selectors;
+	info->max_active_queries = block->num_counters;
+	return 1;
+}
+
+void r600_perfcounters_destroy(struct r600_common_screen *rscreen)
+{
+	if (rscreen->perfcounters)
+		rscreen->perfcounters->cleanup(rscreen);
+}
+
+bool r600_perfcounters_init(struct r600_perfcounters *pc,
+			    unsigned num_blocks)
+{
+	pc->blocks = CALLOC(num_blocks, sizeof(struct r600_perfcounter_block));
+	if (!pc->blocks)
+		return false;
+
+	pc->separate_se = debug_get_bool_option("RADEON_PC_SEPARATE_SE", false);
+	pc->separate_instance = debug_get_bool_option("RADEON_PC_SEPARATE_INSTANCE", false);
+
+	return true;
+}
+
+void r600_perfcounters_add_block(struct r600_common_screen *rscreen,
+				 struct r600_perfcounters *pc,
+				 const char *name, unsigned flags,
+				 unsigned counters, unsigned selectors,
+				 unsigned instances, void *data)
+{
+	struct r600_perfcounter_block *block = &pc->blocks[pc->num_blocks];
+
+	assert(counters <= R600_QUERY_MAX_COUNTERS);
+
+	block->basename = name;
+	block->flags = flags;
+	block->num_counters = counters;
+	block->num_selectors = selectors;
+	block->num_instances = MAX2(instances, 1);
+	block->data = data;
+
+	if (pc->separate_se && (block->flags & R600_PC_BLOCK_SE))
+		block->flags |= R600_PC_BLOCK_SE_GROUPS;
+	if (pc->separate_instance && block->num_instances > 1)
+		block->flags |= R600_PC_BLOCK_INSTANCE_GROUPS;
+
+	if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS) {
+		block->num_groups = block->num_instances;
+	} else {
+		block->num_groups = 1;
+	}
+
+	if (block->flags & R600_PC_BLOCK_SE_GROUPS)
+		block->num_groups *= rscreen->info.max_se;
+	if (block->flags & R600_PC_BLOCK_SHADER)
+		block->num_groups *= pc->num_shader_types;
+
+	++pc->num_blocks;
+	pc->num_groups += block->num_groups;
+}
+
+void r600_perfcounters_do_destroy(struct r600_perfcounters *pc)
+{
+	unsigned i;
+
+	for (i = 0; i < pc->num_blocks; ++i) {
+		FREE(pc->blocks[i].group_names);
+		FREE(pc->blocks[i].selector_names);
+	}
+	FREE(pc->blocks);
+	FREE(pc);
+}
diff --git a/src/gallium/drivers/r600/r600_pipe.c b/src/gallium/drivers/r600/r600_pipe.c
index 62839d4c198..9ed7a17c5c7 100644
--- a/src/gallium/drivers/r600/r600_pipe.c
+++ b/src/gallium/drivers/r600/r600_pipe.c
@@ -37,8 +37,8 @@
 #include "util/u_math.h"
 #include "vl/vl_decoder.h"
 #include "vl/vl_video_buffer.h"
-#include "radeon/radeon_video.h"
-#include "radeon/radeon_uvd.h"
+#include "radeon_video.h"
+#include "radeon_uvd.h"
 #include "os/os_time.h"
 
 static const struct debug_named_value r600_debug_options[] = {
diff --git a/src/gallium/drivers/r600/r600_pipe.h b/src/gallium/drivers/r600/r600_pipe.h
index c9294a7c58d..0d2551ac566 100644
--- a/src/gallium/drivers/r600/r600_pipe.h
+++ b/src/gallium/drivers/r600/r600_pipe.h
@@ -26,8 +26,8 @@
 #ifndef R600_PIPE_H
 #define R600_PIPE_H
 
-#include "radeon/r600_pipe_common.h"
-#include "radeon/r600_cs.h"
+#include "r600_pipe_common.h"
+#include "r600_cs.h"
 #include "r600_public.h"
 #include "pipe/p_defines.h"
 
diff --git a/src/gallium/drivers/r600/r600_pipe_common.c b/src/gallium/drivers/r600/r600_pipe_common.c
new file mode 100644
index 00000000000..0ab8113dc87
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_pipe_common.c
@@ -0,0 +1,1622 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors: Marek Olšák <[email protected]>
+ *
+ */
+
+#include "r600_pipe_common.h"
+#include "r600_cs.h"
+#include "tgsi/tgsi_parse.h"
+#include "util/list.h"
+#include "util/u_draw_quad.h"
+#include "util/u_memory.h"
+#include "util/u_format_s3tc.h"
+#include "util/u_upload_mgr.h"
+#include "os/os_time.h"
+#include "vl/vl_decoder.h"
+#include "vl/vl_video_buffer.h"
+#include "radeon_video.h"
+#include <inttypes.h>
+#include <sys/utsname.h>
+
+#ifndef HAVE_LLVM
+#define HAVE_LLVM 0
+#endif
+
+#if HAVE_LLVM
+#include <llvm-c/TargetMachine.h>
+#endif
+
+#ifndef MESA_LLVM_VERSION_PATCH
+#define MESA_LLVM_VERSION_PATCH 0
+#endif
+
+struct r600_multi_fence {
+	struct pipe_reference reference;
+	struct pipe_fence_handle *gfx;
+	struct pipe_fence_handle *sdma;
+
+	/* If the context wasn't flushed at fence creation, this is non-NULL. */
+	struct {
+		struct r600_common_context *ctx;
+		unsigned ib_index;
+	} gfx_unflushed;
+};
+
+/*
+ * shader binary helpers.
+ */
+void radeon_shader_binary_init(struct ac_shader_binary *b)
+{
+	memset(b, 0, sizeof(*b));
+}
+
+void radeon_shader_binary_clean(struct ac_shader_binary *b)
+{
+	if (!b)
+		return;
+	FREE(b->code);
+	FREE(b->config);
+	FREE(b->rodata);
+	FREE(b->global_symbol_offsets);
+	FREE(b->relocs);
+	FREE(b->disasm_string);
+	FREE(b->llvm_ir_string);
+}
+
+/*
+ * pipe_context
+ */
+
+/**
+ * Write an EOP event.
+ *
+ * \param event		EVENT_TYPE_*
+ * \param event_flags	Optional cache flush flags (TC)
+ * \param data_sel	1 = fence, 3 = timestamp
+ * \param buf		Buffer
+ * \param va		GPU address
+ * \param old_value	Previous fence value (for a bug workaround)
+ * \param new_value	Fence value to write for this event.
+ */
+void r600_gfx_write_event_eop(struct r600_common_context *ctx,
+			      unsigned event, unsigned event_flags,
+			      unsigned data_sel,
+			      struct r600_resource *buf, uint64_t va,
+			      uint32_t new_fence, unsigned query_type)
+{
+	struct radeon_winsys_cs *cs = ctx->gfx.cs;
+	unsigned op = EVENT_TYPE(event) |
+		      EVENT_INDEX(5) |
+		      event_flags;
+	unsigned sel = EOP_DATA_SEL(data_sel);
+
+	/* Wait for write confirmation before writing data, but don't send
+	 * an interrupt. */
+	if (ctx->chip_class >= SI && data_sel != EOP_DATA_SEL_DISCARD)
+		sel |= EOP_INT_SEL(EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM);
+
+	if (ctx->chip_class >= GFX9) {
+		/* A ZPASS_DONE or PIXEL_STAT_DUMP_EVENT (of the DB occlusion
+		 * counters) must immediately precede every timestamp event to
+		 * prevent a GPU hang on GFX9.
+		 *
+		 * Occlusion queries don't need to do it here, because they
+		 * always do ZPASS_DONE before the timestamp.
+		 */
+		if (ctx->chip_class == GFX9 &&
+		    query_type != PIPE_QUERY_OCCLUSION_COUNTER &&
+		    query_type != PIPE_QUERY_OCCLUSION_PREDICATE) {
+			struct r600_resource *scratch = ctx->eop_bug_scratch;
+
+			assert(16 * ctx->screen->info.num_render_backends <=
+			       scratch->b.b.width0);
+			radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
+			radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1));
+			radeon_emit(cs, scratch->gpu_address);
+			radeon_emit(cs, scratch->gpu_address >> 32);
+
+			radeon_add_to_buffer_list(ctx, &ctx->gfx, scratch,
+						  RADEON_USAGE_WRITE, RADEON_PRIO_QUERY);
+		}
+
+		radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 6, 0));
+		radeon_emit(cs, op);
+		radeon_emit(cs, sel);
+		radeon_emit(cs, va);		/* address lo */
+		radeon_emit(cs, va >> 32);	/* address hi */
+		radeon_emit(cs, new_fence);	/* immediate data lo */
+		radeon_emit(cs, 0); /* immediate data hi */
+		radeon_emit(cs, 0); /* unused */
+	} else {
+		if (ctx->chip_class == CIK ||
+		    ctx->chip_class == VI) {
+			struct r600_resource *scratch = ctx->eop_bug_scratch;
+			uint64_t va = scratch->gpu_address;
+
+			/* Two EOP events are required to make all engines go idle
+			 * (and optional cache flushes executed) before the timestamp
+			 * is written.
+			 */
+			radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, 0));
+			radeon_emit(cs, op);
+			radeon_emit(cs, va);
+			radeon_emit(cs, ((va >> 32) & 0xffff) | sel);
+			radeon_emit(cs, 0); /* immediate data */
+			radeon_emit(cs, 0); /* unused */
+
+			radeon_add_to_buffer_list(ctx, &ctx->gfx, scratch,
+						  RADEON_USAGE_WRITE, RADEON_PRIO_QUERY);
+		}
+
+		radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, 0));
+		radeon_emit(cs, op);
+		radeon_emit(cs, va);
+		radeon_emit(cs, ((va >> 32) & 0xffff) | sel);
+		radeon_emit(cs, new_fence); /* immediate data */
+		radeon_emit(cs, 0); /* unused */
+	}
+
+	if (buf)
+		r600_emit_reloc(ctx, &ctx->gfx, buf, RADEON_USAGE_WRITE,
+				RADEON_PRIO_QUERY);
+}
+
+unsigned r600_gfx_write_fence_dwords(struct r600_common_screen *screen)
+{
+	unsigned dwords = 6;
+
+	if (screen->chip_class == CIK ||
+	    screen->chip_class == VI)
+		dwords *= 2;
+
+	if (!screen->info.has_virtual_memory)
+		dwords += 2;
+
+	return dwords;
+}
+
+void r600_gfx_wait_fence(struct r600_common_context *ctx,
+			 uint64_t va, uint32_t ref, uint32_t mask)
+{
+	struct radeon_winsys_cs *cs = ctx->gfx.cs;
+
+	radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0));
+	radeon_emit(cs, WAIT_REG_MEM_EQUAL | WAIT_REG_MEM_MEM_SPACE(1));
+	radeon_emit(cs, va);
+	radeon_emit(cs, va >> 32);
+	radeon_emit(cs, ref); /* reference value */
+	radeon_emit(cs, mask); /* mask */
+	radeon_emit(cs, 4); /* poll interval */
+}
+
+void r600_draw_rectangle(struct blitter_context *blitter,
+			 int x1, int y1, int x2, int y2,
+			 float depth, unsigned num_instances,
+			 enum blitter_attrib_type type,
+			 const union blitter_attrib *attrib)
+{
+	struct r600_common_context *rctx =
+		(struct r600_common_context*)util_blitter_get_pipe(blitter);
+	struct pipe_viewport_state viewport;
+	struct pipe_resource *buf = NULL;
+	unsigned offset = 0;
+	float *vb;
+
+	/* Some operations (like color resolve on r6xx) don't work
+	 * with the conventional primitive types.
+	 * One that works is PT_RECTLIST, which we use here. */
+
+	/* setup viewport */
+	viewport.scale[0] = 1.0f;
+	viewport.scale[1] = 1.0f;
+	viewport.scale[2] = 1.0f;
+	viewport.translate[0] = 0.0f;
+	viewport.translate[1] = 0.0f;
+	viewport.translate[2] = 0.0f;
+	rctx->b.set_viewport_states(&rctx->b, 0, 1, &viewport);
+
+	/* Upload vertices. The hw rectangle has only 3 vertices,
+	 * The 4th one is derived from the first 3.
+	 * The vertex specification should match u_blitter's vertex element state. */
+	u_upload_alloc(rctx->b.stream_uploader, 0, sizeof(float) * 24,
+		       rctx->screen->info.tcc_cache_line_size,
+                       &offset, &buf, (void**)&vb);
+	if (!buf)
+		return;
+
+	vb[0] = x1;
+	vb[1] = y1;
+	vb[2] = depth;
+	vb[3] = 1;
+
+	vb[8] = x1;
+	vb[9] = y2;
+	vb[10] = depth;
+	vb[11] = 1;
+
+	vb[16] = x2;
+	vb[17] = y1;
+	vb[18] = depth;
+	vb[19] = 1;
+
+	switch (type) {
+	case UTIL_BLITTER_ATTRIB_COLOR:
+		memcpy(vb+4, attrib->color, sizeof(float)*4);
+		memcpy(vb+12, attrib->color, sizeof(float)*4);
+		memcpy(vb+20, attrib->color, sizeof(float)*4);
+		break;
+	case UTIL_BLITTER_ATTRIB_TEXCOORD_XYZW:
+	case UTIL_BLITTER_ATTRIB_TEXCOORD_XY:
+		vb[6] = vb[14] = vb[22] = attrib->texcoord.z;
+		vb[7] = vb[15] = vb[23] = attrib->texcoord.w;
+		/* fall through */
+		vb[4] = attrib->texcoord.x1;
+		vb[5] = attrib->texcoord.y1;
+		vb[12] = attrib->texcoord.x1;
+		vb[13] = attrib->texcoord.y2;
+		vb[20] = attrib->texcoord.x2;
+		vb[21] = attrib->texcoord.y1;
+		break;
+	default:; /* Nothing to do. */
+	}
+
+	/* draw */
+	struct pipe_vertex_buffer vbuffer = {};
+	vbuffer.buffer.resource = buf;
+	vbuffer.stride = 2 * 4 * sizeof(float); /* vertex size */
+	vbuffer.buffer_offset = offset;
+
+	rctx->b.set_vertex_buffers(&rctx->b, blitter->vb_slot, 1, &vbuffer);
+	util_draw_arrays_instanced(&rctx->b, R600_PRIM_RECTANGLE_LIST, 0, 3,
+				   0, num_instances);
+	pipe_resource_reference(&buf, NULL);
+}
+
+static void r600_dma_emit_wait_idle(struct r600_common_context *rctx)
+{
+	struct radeon_winsys_cs *cs = rctx->dma.cs;
+
+	/* NOP waits for idle on Evergreen and later. */
+	if (rctx->chip_class >= CIK)
+		radeon_emit(cs, 0x00000000); /* NOP */
+	else if (rctx->chip_class >= EVERGREEN)
+		radeon_emit(cs, 0xf0000000); /* NOP */
+	else {
+		/* TODO: R600-R700 should use the FENCE packet.
+		 * CS checker support is required. */
+	}
+}
+
+void r600_need_dma_space(struct r600_common_context *ctx, unsigned num_dw,
+                         struct r600_resource *dst, struct r600_resource *src)
+{
+	uint64_t vram = ctx->dma.cs->used_vram;
+	uint64_t gtt = ctx->dma.cs->used_gart;
+
+	if (dst) {
+		vram += dst->vram_usage;
+		gtt += dst->gart_usage;
+	}
+	if (src) {
+		vram += src->vram_usage;
+		gtt += src->gart_usage;
+	}
+
+	/* Flush the GFX IB if DMA depends on it. */
+	if (radeon_emitted(ctx->gfx.cs, ctx->initial_gfx_cs_size) &&
+	    ((dst &&
+	      ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, dst->buf,
+					       RADEON_USAGE_READWRITE)) ||
+	     (src &&
+	      ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, src->buf,
+					       RADEON_USAGE_WRITE))))
+		ctx->gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
+
+	/* Flush if there's not enough space, or if the memory usage per IB
+	 * is too large.
+	 *
+	 * IBs using too little memory are limited by the IB submission overhead.
+	 * IBs using too much memory are limited by the kernel/TTM overhead.
+	 * Too long IBs create CPU-GPU pipeline bubbles and add latency.
+	 *
+	 * This heuristic makes sure that DMA requests are executed
+	 * very soon after the call is made and lowers memory usage.
+	 * It improves texture upload performance by keeping the DMA
+	 * engine busy while uploads are being submitted.
+	 */
+	num_dw++; /* for emit_wait_idle below */
+	if (!ctx->ws->cs_check_space(ctx->dma.cs, num_dw) ||
+	    ctx->dma.cs->used_vram + ctx->dma.cs->used_gart > 64 * 1024 * 1024 ||
+	    !radeon_cs_memory_below_limit(ctx->screen, ctx->dma.cs, vram, gtt)) {
+		ctx->dma.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
+		assert((num_dw + ctx->dma.cs->current.cdw) <= ctx->dma.cs->current.max_dw);
+	}
+
+	/* Wait for idle if either buffer has been used in the IB before to
+	 * prevent read-after-write hazards.
+	 */
+	if ((dst &&
+	     ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, dst->buf,
+					      RADEON_USAGE_READWRITE)) ||
+	    (src &&
+	     ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, src->buf,
+					      RADEON_USAGE_WRITE)))
+		r600_dma_emit_wait_idle(ctx);
+
+	/* If GPUVM is not supported, the CS checker needs 2 entries
+	 * in the buffer list per packet, which has to be done manually.
+	 */
+	if (ctx->screen->info.has_virtual_memory) {
+		if (dst)
+			radeon_add_to_buffer_list(ctx, &ctx->dma, dst,
+						  RADEON_USAGE_WRITE,
+						  RADEON_PRIO_SDMA_BUFFER);
+		if (src)
+			radeon_add_to_buffer_list(ctx, &ctx->dma, src,
+						  RADEON_USAGE_READ,
+						  RADEON_PRIO_SDMA_BUFFER);
+	}
+
+	/* this function is called before all DMA calls, so increment this. */
+	ctx->num_dma_calls++;
+}
+
+static void r600_memory_barrier(struct pipe_context *ctx, unsigned flags)
+{
+}
+
+void r600_preflush_suspend_features(struct r600_common_context *ctx)
+{
+	/* suspend queries */
+	if (!LIST_IS_EMPTY(&ctx->active_queries))
+		r600_suspend_queries(ctx);
+
+	ctx->streamout.suspended = false;
+	if (ctx->streamout.begin_emitted) {
+		r600_emit_streamout_end(ctx);
+		ctx->streamout.suspended = true;
+	}
+}
+
+void r600_postflush_resume_features(struct r600_common_context *ctx)
+{
+	if (ctx->streamout.suspended) {
+		ctx->streamout.append_bitmask = ctx->streamout.enabled_mask;
+		r600_streamout_buffers_dirty(ctx);
+	}
+
+	/* resume queries */
+	if (!LIST_IS_EMPTY(&ctx->active_queries))
+		r600_resume_queries(ctx);
+}
+
+static void r600_add_fence_dependency(struct r600_common_context *rctx,
+				      struct pipe_fence_handle *fence)
+{
+	struct radeon_winsys *ws = rctx->ws;
+
+	if (rctx->dma.cs)
+		ws->cs_add_fence_dependency(rctx->dma.cs, fence);
+	ws->cs_add_fence_dependency(rctx->gfx.cs, fence);
+}
+
+static void r600_fence_server_sync(struct pipe_context *ctx,
+				   struct pipe_fence_handle *fence)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct r600_multi_fence *rfence = (struct r600_multi_fence *)fence;
+
+	/* Only amdgpu needs to handle fence dependencies (for fence imports).
+	 * radeon synchronizes all rings by default and will not implement
+	 * fence imports.
+	 */
+	if (rctx->screen->info.drm_major == 2)
+		return;
+
+	/* Only imported fences need to be handled by fence_server_sync,
+	 * because the winsys handles synchronizations automatically for BOs
+	 * within the process.
+	 *
+	 * Simply skip unflushed fences here, and the winsys will drop no-op
+	 * dependencies (i.e. dependencies within the same ring).
+	 */
+	if (rfence->gfx_unflushed.ctx)
+		return;
+
+	/* All unflushed commands will not start execution before
+	 * this fence dependency is signalled.
+	 *
+	 * Should we flush the context to allow more GPU parallelism?
+	 */
+	if (rfence->sdma)
+		r600_add_fence_dependency(rctx, rfence->sdma);
+	if (rfence->gfx)
+		r600_add_fence_dependency(rctx, rfence->gfx);
+}
+
+static void r600_flush_from_st(struct pipe_context *ctx,
+			       struct pipe_fence_handle **fence,
+			       unsigned flags)
+{
+	struct pipe_screen *screen = ctx->screen;
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct radeon_winsys *ws = rctx->ws;
+	struct pipe_fence_handle *gfx_fence = NULL;
+	struct pipe_fence_handle *sdma_fence = NULL;
+	bool deferred_fence = false;
+	unsigned rflags = RADEON_FLUSH_ASYNC;
+
+	if (flags & PIPE_FLUSH_END_OF_FRAME)
+		rflags |= RADEON_FLUSH_END_OF_FRAME;
+
+	/* DMA IBs are preambles to gfx IBs, therefore must be flushed first. */
+	if (rctx->dma.cs)
+		rctx->dma.flush(rctx, rflags, fence ? &sdma_fence : NULL);
+
+	if (!radeon_emitted(rctx->gfx.cs, rctx->initial_gfx_cs_size)) {
+		if (fence)
+			ws->fence_reference(&gfx_fence, rctx->last_gfx_fence);
+		if (!(flags & PIPE_FLUSH_DEFERRED))
+			ws->cs_sync_flush(rctx->gfx.cs);
+	} else {
+		/* Instead of flushing, create a deferred fence. Constraints:
+		 * - The state tracker must allow a deferred flush.
+		 * - The state tracker must request a fence.
+		 * Thread safety in fence_finish must be ensured by the state tracker.
+		 */
+		if (flags & PIPE_FLUSH_DEFERRED && fence) {
+			gfx_fence = rctx->ws->cs_get_next_fence(rctx->gfx.cs);
+			deferred_fence = true;
+		} else {
+			rctx->gfx.flush(rctx, rflags, fence ? &gfx_fence : NULL);
+		}
+	}
+
+	/* Both engines can signal out of order, so we need to keep both fences. */
+	if (fence) {
+		struct r600_multi_fence *multi_fence =
+			CALLOC_STRUCT(r600_multi_fence);
+		if (!multi_fence) {
+			ws->fence_reference(&sdma_fence, NULL);
+			ws->fence_reference(&gfx_fence, NULL);
+			goto finish;
+		}
+
+		multi_fence->reference.count = 1;
+		/* If both fences are NULL, fence_finish will always return true. */
+		multi_fence->gfx = gfx_fence;
+		multi_fence->sdma = sdma_fence;
+
+		if (deferred_fence) {
+			multi_fence->gfx_unflushed.ctx = rctx;
+			multi_fence->gfx_unflushed.ib_index = rctx->num_gfx_cs_flushes;
+		}
+
+		screen->fence_reference(screen, fence, NULL);
+		*fence = (struct pipe_fence_handle*)multi_fence;
+	}
+finish:
+	if (!(flags & PIPE_FLUSH_DEFERRED)) {
+		if (rctx->dma.cs)
+			ws->cs_sync_flush(rctx->dma.cs);
+		ws->cs_sync_flush(rctx->gfx.cs);
+	}
+}
+
+static void r600_flush_dma_ring(void *ctx, unsigned flags,
+				struct pipe_fence_handle **fence)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct radeon_winsys_cs *cs = rctx->dma.cs;
+	struct radeon_saved_cs saved;
+	bool check_vm =
+		(rctx->screen->debug_flags & DBG_CHECK_VM) &&
+		rctx->check_vm_faults;
+
+	if (!radeon_emitted(cs, 0)) {
+		if (fence)
+			rctx->ws->fence_reference(fence, rctx->last_sdma_fence);
+		return;
+	}
+
+	if (check_vm)
+		radeon_save_cs(rctx->ws, cs, &saved, true);
+
+	rctx->ws->cs_flush(cs, flags, &rctx->last_sdma_fence);
+	if (fence)
+		rctx->ws->fence_reference(fence, rctx->last_sdma_fence);
+
+	if (check_vm) {
+		/* Use conservative timeout 800ms, after which we won't wait any
+		 * longer and assume the GPU is hung.
+		 */
+		rctx->ws->fence_wait(rctx->ws, rctx->last_sdma_fence, 800*1000*1000);
+
+		rctx->check_vm_faults(rctx, &saved, RING_DMA);
+		radeon_clear_saved_cs(&saved);
+	}
+}
+
+/**
+ * Store a linearized copy of all chunks of \p cs together with the buffer
+ * list in \p saved.
+ */
+void radeon_save_cs(struct radeon_winsys *ws, struct radeon_winsys_cs *cs,
+		    struct radeon_saved_cs *saved, bool get_buffer_list)
+{
+	uint32_t *buf;
+	unsigned i;
+
+	/* Save the IB chunks. */
+	saved->num_dw = cs->prev_dw + cs->current.cdw;
+	saved->ib = MALLOC(4 * saved->num_dw);
+	if (!saved->ib)
+		goto oom;
+
+	buf = saved->ib;
+	for (i = 0; i < cs->num_prev; ++i) {
+		memcpy(buf, cs->prev[i].buf, cs->prev[i].cdw * 4);
+		buf += cs->prev[i].cdw;
+	}
+	memcpy(buf, cs->current.buf, cs->current.cdw * 4);
+
+	if (!get_buffer_list)
+		return;
+
+	/* Save the buffer list. */
+	saved->bo_count = ws->cs_get_buffer_list(cs, NULL);
+	saved->bo_list = CALLOC(saved->bo_count,
+				sizeof(saved->bo_list[0]));
+	if (!saved->bo_list) {
+		FREE(saved->ib);
+		goto oom;
+	}
+	ws->cs_get_buffer_list(cs, saved->bo_list);
+
+	return;
+
+oom:
+	fprintf(stderr, "%s: out of memory\n", __func__);
+	memset(saved, 0, sizeof(*saved));
+}
+
+void radeon_clear_saved_cs(struct radeon_saved_cs *saved)
+{
+	FREE(saved->ib);
+	FREE(saved->bo_list);
+
+	memset(saved, 0, sizeof(*saved));
+}
+
+static enum pipe_reset_status r600_get_reset_status(struct pipe_context *ctx)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	unsigned latest = rctx->ws->query_value(rctx->ws,
+						RADEON_GPU_RESET_COUNTER);
+
+	if (rctx->gpu_reset_counter == latest)
+		return PIPE_NO_RESET;
+
+	rctx->gpu_reset_counter = latest;
+	return PIPE_UNKNOWN_CONTEXT_RESET;
+}
+
+static void r600_set_debug_callback(struct pipe_context *ctx,
+				    const struct pipe_debug_callback *cb)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+
+	if (cb)
+		rctx->debug = *cb;
+	else
+		memset(&rctx->debug, 0, sizeof(rctx->debug));
+}
+
+static void r600_set_device_reset_callback(struct pipe_context *ctx,
+					   const struct pipe_device_reset_callback *cb)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+
+	if (cb)
+		rctx->device_reset_callback = *cb;
+	else
+		memset(&rctx->device_reset_callback, 0,
+		       sizeof(rctx->device_reset_callback));
+}
+
+bool r600_check_device_reset(struct r600_common_context *rctx)
+{
+	enum pipe_reset_status status;
+
+	if (!rctx->device_reset_callback.reset)
+		return false;
+
+	if (!rctx->b.get_device_reset_status)
+		return false;
+
+	status = rctx->b.get_device_reset_status(&rctx->b);
+	if (status == PIPE_NO_RESET)
+		return false;
+
+	rctx->device_reset_callback.reset(rctx->device_reset_callback.data, status);
+	return true;
+}
+
+static void r600_dma_clear_buffer_fallback(struct pipe_context *ctx,
+					   struct pipe_resource *dst,
+					   uint64_t offset, uint64_t size,
+					   unsigned value)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+
+	rctx->clear_buffer(ctx, dst, offset, size, value, R600_COHERENCY_NONE);
+}
+
+static bool r600_resource_commit(struct pipe_context *pctx,
+				 struct pipe_resource *resource,
+				 unsigned level, struct pipe_box *box,
+				 bool commit)
+{
+	struct r600_common_context *ctx = (struct r600_common_context *)pctx;
+	struct r600_resource *res = r600_resource(resource);
+
+	/*
+	 * Since buffer commitment changes cannot be pipelined, we need to
+	 * (a) flush any pending commands that refer to the buffer we're about
+	 *     to change, and
+	 * (b) wait for threaded submit to finish, including those that were
+	 *     triggered by some other, earlier operation.
+	 */
+	if (radeon_emitted(ctx->gfx.cs, ctx->initial_gfx_cs_size) &&
+	    ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs,
+					     res->buf, RADEON_USAGE_READWRITE)) {
+		ctx->gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
+	}
+	if (radeon_emitted(ctx->dma.cs, 0) &&
+	    ctx->ws->cs_is_buffer_referenced(ctx->dma.cs,
+					     res->buf, RADEON_USAGE_READWRITE)) {
+		ctx->dma.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
+	}
+
+	ctx->ws->cs_sync_flush(ctx->dma.cs);
+	ctx->ws->cs_sync_flush(ctx->gfx.cs);
+
+	assert(resource->target == PIPE_BUFFER);
+
+	return ctx->ws->buffer_commit(res->buf, box->x, box->width, commit);
+}
+
+bool r600_common_context_init(struct r600_common_context *rctx,
+			      struct r600_common_screen *rscreen,
+			      unsigned context_flags)
+{
+	slab_create_child(&rctx->pool_transfers, &rscreen->pool_transfers);
+	slab_create_child(&rctx->pool_transfers_unsync, &rscreen->pool_transfers);
+
+	rctx->screen = rscreen;
+	rctx->ws = rscreen->ws;
+	rctx->family = rscreen->family;
+	rctx->chip_class = rscreen->chip_class;
+
+	rctx->b.invalidate_resource = r600_invalidate_resource;
+	rctx->b.resource_commit = r600_resource_commit;
+	rctx->b.transfer_map = u_transfer_map_vtbl;
+	rctx->b.transfer_flush_region = u_transfer_flush_region_vtbl;
+	rctx->b.transfer_unmap = u_transfer_unmap_vtbl;
+	rctx->b.texture_subdata = u_default_texture_subdata;
+	rctx->b.memory_barrier = r600_memory_barrier;
+	rctx->b.flush = r600_flush_from_st;
+	rctx->b.set_debug_callback = r600_set_debug_callback;
+	rctx->b.fence_server_sync = r600_fence_server_sync;
+	rctx->dma_clear_buffer = r600_dma_clear_buffer_fallback;
+
+	/* evergreen_compute.c has a special codepath for global buffers.
+	 * Everything else can use the direct path.
+	 */
+	if ((rscreen->chip_class == EVERGREEN || rscreen->chip_class == CAYMAN) &&
+	    (context_flags & PIPE_CONTEXT_COMPUTE_ONLY))
+		rctx->b.buffer_subdata = u_default_buffer_subdata;
+	else
+		rctx->b.buffer_subdata = r600_buffer_subdata;
+
+	if (rscreen->info.drm_major == 2 && rscreen->info.drm_minor >= 43) {
+		rctx->b.get_device_reset_status = r600_get_reset_status;
+		rctx->gpu_reset_counter =
+			rctx->ws->query_value(rctx->ws,
+					      RADEON_GPU_RESET_COUNTER);
+	}
+
+	rctx->b.set_device_reset_callback = r600_set_device_reset_callback;
+
+	r600_init_context_texture_functions(rctx);
+	r600_init_viewport_functions(rctx);
+	r600_streamout_init(rctx);
+	r600_query_init(rctx);
+	cayman_init_msaa(&rctx->b);
+
+	if (rctx->chip_class == CIK ||
+	    rctx->chip_class == VI ||
+	    rctx->chip_class == GFX9) {
+		rctx->eop_bug_scratch = (struct r600_resource*)
+			pipe_buffer_create(&rscreen->b, 0, PIPE_USAGE_DEFAULT,
+					   16 * rscreen->info.num_render_backends);
+		if (!rctx->eop_bug_scratch)
+			return false;
+	}
+
+	rctx->allocator_zeroed_memory =
+		u_suballocator_create(&rctx->b, rscreen->info.gart_page_size,
+				      0, PIPE_USAGE_DEFAULT, 0, true);
+	if (!rctx->allocator_zeroed_memory)
+		return false;
+
+	rctx->b.stream_uploader = u_upload_create(&rctx->b, 1024 * 1024,
+						  0, PIPE_USAGE_STREAM);
+	if (!rctx->b.stream_uploader)
+		return false;
+
+	rctx->b.const_uploader = u_upload_create(&rctx->b, 128 * 1024,
+						 0, PIPE_USAGE_DEFAULT);
+	if (!rctx->b.const_uploader)
+		return false;
+
+	rctx->ctx = rctx->ws->ctx_create(rctx->ws);
+	if (!rctx->ctx)
+		return false;
+
+	if (rscreen->info.num_sdma_rings && !(rscreen->debug_flags & DBG_NO_ASYNC_DMA)) {
+		rctx->dma.cs = rctx->ws->cs_create(rctx->ctx, RING_DMA,
+						   r600_flush_dma_ring,
+						   rctx);
+		rctx->dma.flush = r600_flush_dma_ring;
+	}
+
+	return true;
+}
+
+void r600_common_context_cleanup(struct r600_common_context *rctx)
+{
+	unsigned i,j;
+
+	/* Release DCC stats. */
+	for (i = 0; i < ARRAY_SIZE(rctx->dcc_stats); i++) {
+		assert(!rctx->dcc_stats[i].query_active);
+
+		for (j = 0; j < ARRAY_SIZE(rctx->dcc_stats[i].ps_stats); j++)
+			if (rctx->dcc_stats[i].ps_stats[j])
+				rctx->b.destroy_query(&rctx->b,
+						      rctx->dcc_stats[i].ps_stats[j]);
+
+		r600_texture_reference(&rctx->dcc_stats[i].tex, NULL);
+	}
+
+	if (rctx->query_result_shader)
+		rctx->b.delete_compute_state(&rctx->b, rctx->query_result_shader);
+
+	if (rctx->gfx.cs)
+		rctx->ws->cs_destroy(rctx->gfx.cs);
+	if (rctx->dma.cs)
+		rctx->ws->cs_destroy(rctx->dma.cs);
+	if (rctx->ctx)
+		rctx->ws->ctx_destroy(rctx->ctx);
+
+	if (rctx->b.stream_uploader)
+		u_upload_destroy(rctx->b.stream_uploader);
+	if (rctx->b.const_uploader)
+		u_upload_destroy(rctx->b.const_uploader);
+
+	slab_destroy_child(&rctx->pool_transfers);
+	slab_destroy_child(&rctx->pool_transfers_unsync);
+
+	if (rctx->allocator_zeroed_memory) {
+		u_suballocator_destroy(rctx->allocator_zeroed_memory);
+	}
+	rctx->ws->fence_reference(&rctx->last_gfx_fence, NULL);
+	rctx->ws->fence_reference(&rctx->last_sdma_fence, NULL);
+	r600_resource_reference(&rctx->eop_bug_scratch, NULL);
+}
+
+/*
+ * pipe_screen
+ */
+
+static const struct debug_named_value common_debug_options[] = {
+	/* logging */
+	{ "tex", DBG_TEX, "Print texture info" },
+	{ "nir", DBG_NIR, "Enable experimental NIR shaders" },
+	{ "compute", DBG_COMPUTE, "Print compute info" },
+	{ "vm", DBG_VM, "Print virtual addresses when creating resources" },
+	{ "info", DBG_INFO, "Print driver information" },
+
+	/* shaders */
+	{ "fs", DBG_FS, "Print fetch shaders" },
+	{ "vs", DBG_VS, "Print vertex shaders" },
+	{ "gs", DBG_GS, "Print geometry shaders" },
+	{ "ps", DBG_PS, "Print pixel shaders" },
+	{ "cs", DBG_CS, "Print compute shaders" },
+	{ "tcs", DBG_TCS, "Print tessellation control shaders" },
+	{ "tes", DBG_TES, "Print tessellation evaluation shaders" },
+	{ "noir", DBG_NO_IR, "Don't print the LLVM IR"},
+	{ "notgsi", DBG_NO_TGSI, "Don't print the TGSI"},
+	{ "noasm", DBG_NO_ASM, "Don't print disassembled shaders"},
+	{ "preoptir", DBG_PREOPT_IR, "Print the LLVM IR before initial optimizations" },
+	{ "checkir", DBG_CHECK_IR, "Enable additional sanity checks on shader IR" },
+	{ "nooptvariant", DBG_NO_OPT_VARIANT, "Disable compiling optimized shader variants." },
+
+	{ "testdma", DBG_TEST_DMA, "Invoke SDMA tests and exit." },
+	{ "testvmfaultcp", DBG_TEST_VMFAULT_CP, "Invoke a CP VM fault test and exit." },
+	{ "testvmfaultsdma", DBG_TEST_VMFAULT_SDMA, "Invoke a SDMA VM fault test and exit." },
+	{ "testvmfaultshader", DBG_TEST_VMFAULT_SHADER, "Invoke a shader VM fault test and exit." },
+
+	/* features */
+	{ "nodma", DBG_NO_ASYNC_DMA, "Disable asynchronous DMA" },
+	{ "nohyperz", DBG_NO_HYPERZ, "Disable Hyper-Z" },
+	/* GL uses the word INVALIDATE, gallium uses the word DISCARD */
+	{ "noinvalrange", DBG_NO_DISCARD_RANGE, "Disable handling of INVALIDATE_RANGE map flags" },
+	{ "no2d", DBG_NO_2D_TILING, "Disable 2D tiling" },
+	{ "notiling", DBG_NO_TILING, "Disable tiling" },
+	{ "switch_on_eop", DBG_SWITCH_ON_EOP, "Program WD/IA to switch on end-of-packet." },
+	{ "forcedma", DBG_FORCE_DMA, "Use asynchronous DMA for all operations when possible." },
+	{ "precompile", DBG_PRECOMPILE, "Compile one shader variant at shader creation." },
+	{ "nowc", DBG_NO_WC, "Disable GTT write combining" },
+	{ "check_vm", DBG_CHECK_VM, "Check VM faults and dump debug info." },
+	{ "nodcc", DBG_NO_DCC, "Disable DCC." },
+	{ "nodccclear", DBG_NO_DCC_CLEAR, "Disable DCC fast clear." },
+	{ "norbplus", DBG_NO_RB_PLUS, "Disable RB+." },
+	{ "sisched", DBG_SI_SCHED, "Enable LLVM SI Machine Instruction Scheduler." },
+	{ "mono", DBG_MONOLITHIC_SHADERS, "Use old-style monolithic shaders compiled on demand" },
+	{ "unsafemath", DBG_UNSAFE_MATH, "Enable unsafe math shader optimizations" },
+	{ "nodccfb", DBG_NO_DCC_FB, "Disable separate DCC on the main framebuffer" },
+	{ "nodpbb", DBG_NO_DPBB, "Disable DPBB." },
+	{ "nodfsm", DBG_NO_DFSM, "Disable DFSM." },
+
+	DEBUG_NAMED_VALUE_END /* must be last */
+};
+
+static const char* r600_get_vendor(struct pipe_screen* pscreen)
+{
+	return "X.Org";
+}
+
+static const char* r600_get_device_vendor(struct pipe_screen* pscreen)
+{
+	return "AMD";
+}
+
+static const char *r600_get_marketing_name(struct radeon_winsys *ws)
+{
+	if (!ws->get_chip_name)
+		return NULL;
+	return ws->get_chip_name(ws);
+}
+
+static const char *r600_get_family_name(const struct r600_common_screen *rscreen)
+{
+	switch (rscreen->info.family) {
+	case CHIP_R600: return "AMD R600";
+	case CHIP_RV610: return "AMD RV610";
+	case CHIP_RV630: return "AMD RV630";
+	case CHIP_RV670: return "AMD RV670";
+	case CHIP_RV620: return "AMD RV620";
+	case CHIP_RV635: return "AMD RV635";
+	case CHIP_RS780: return "AMD RS780";
+	case CHIP_RS880: return "AMD RS880";
+	case CHIP_RV770: return "AMD RV770";
+	case CHIP_RV730: return "AMD RV730";
+	case CHIP_RV710: return "AMD RV710";
+	case CHIP_RV740: return "AMD RV740";
+	case CHIP_CEDAR: return "AMD CEDAR";
+	case CHIP_REDWOOD: return "AMD REDWOOD";
+	case CHIP_JUNIPER: return "AMD JUNIPER";
+	case CHIP_CYPRESS: return "AMD CYPRESS";
+	case CHIP_HEMLOCK: return "AMD HEMLOCK";
+	case CHIP_PALM: return "AMD PALM";
+	case CHIP_SUMO: return "AMD SUMO";
+	case CHIP_SUMO2: return "AMD SUMO2";
+	case CHIP_BARTS: return "AMD BARTS";
+	case CHIP_TURKS: return "AMD TURKS";
+	case CHIP_CAICOS: return "AMD CAICOS";
+	case CHIP_CAYMAN: return "AMD CAYMAN";
+	case CHIP_ARUBA: return "AMD ARUBA";
+	case CHIP_TAHITI: return "AMD TAHITI";
+	case CHIP_PITCAIRN: return "AMD PITCAIRN";
+	case CHIP_VERDE: return "AMD CAPE VERDE";
+	case CHIP_OLAND: return "AMD OLAND";
+	case CHIP_HAINAN: return "AMD HAINAN";
+	case CHIP_BONAIRE: return "AMD BONAIRE";
+	case CHIP_KAVERI: return "AMD KAVERI";
+	case CHIP_KABINI: return "AMD KABINI";
+	case CHIP_HAWAII: return "AMD HAWAII";
+	case CHIP_MULLINS: return "AMD MULLINS";
+	case CHIP_TONGA: return "AMD TONGA";
+	case CHIP_ICELAND: return "AMD ICELAND";
+	case CHIP_CARRIZO: return "AMD CARRIZO";
+	case CHIP_FIJI: return "AMD FIJI";
+	case CHIP_POLARIS10: return "AMD POLARIS10";
+	case CHIP_POLARIS11: return "AMD POLARIS11";
+	case CHIP_POLARIS12: return "AMD POLARIS12";
+	case CHIP_STONEY: return "AMD STONEY";
+	case CHIP_VEGA10: return "AMD VEGA10";
+	case CHIP_RAVEN: return "AMD RAVEN";
+	default: return "AMD unknown";
+	}
+}
+
+static void r600_disk_cache_create(struct r600_common_screen *rscreen)
+{
+	/* Don't use the cache if shader dumping is enabled. */
+	if (rscreen->debug_flags & DBG_ALL_SHADERS)
+		return;
+
+	uint32_t mesa_timestamp;
+	if (disk_cache_get_function_timestamp(r600_disk_cache_create,
+					      &mesa_timestamp)) {
+		char *timestamp_str;
+		int res = -1;
+		if (rscreen->chip_class < SI) {
+			res = asprintf(&timestamp_str, "%u",mesa_timestamp);
+		}
+#if HAVE_LLVM
+		else {
+			uint32_t llvm_timestamp;
+			if (disk_cache_get_function_timestamp(LLVMInitializeAMDGPUTargetInfo,
+							      &llvm_timestamp)) {
+				res = asprintf(&timestamp_str, "%u_%u",
+					       mesa_timestamp, llvm_timestamp);
+			}
+		}
+#endif
+		if (res != -1) {
+			/* These flags affect shader compilation. */
+			uint64_t shader_debug_flags =
+				rscreen->debug_flags &
+				(DBG_FS_CORRECT_DERIVS_AFTER_KILL |
+				 DBG_SI_SCHED |
+				 DBG_UNSAFE_MATH);
+
+			rscreen->disk_shader_cache =
+				disk_cache_create(r600_get_family_name(rscreen),
+						  timestamp_str,
+						  shader_debug_flags);
+			free(timestamp_str);
+		}
+	}
+}
+
+static struct disk_cache *r600_get_disk_shader_cache(struct pipe_screen *pscreen)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)pscreen;
+	return rscreen->disk_shader_cache;
+}
+
+static const char* r600_get_name(struct pipe_screen* pscreen)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)pscreen;
+
+	return rscreen->renderer_string;
+}
+
+static float r600_get_paramf(struct pipe_screen* pscreen,
+			     enum pipe_capf param)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen *)pscreen;
+
+	switch (param) {
+	case PIPE_CAPF_MAX_LINE_WIDTH:
+	case PIPE_CAPF_MAX_LINE_WIDTH_AA:
+	case PIPE_CAPF_MAX_POINT_WIDTH:
+	case PIPE_CAPF_MAX_POINT_WIDTH_AA:
+		if (rscreen->family >= CHIP_CEDAR)
+			return 16384.0f;
+		else
+			return 8192.0f;
+	case PIPE_CAPF_MAX_TEXTURE_ANISOTROPY:
+		return 16.0f;
+	case PIPE_CAPF_MAX_TEXTURE_LOD_BIAS:
+		return 16.0f;
+	case PIPE_CAPF_GUARD_BAND_LEFT:
+	case PIPE_CAPF_GUARD_BAND_TOP:
+	case PIPE_CAPF_GUARD_BAND_RIGHT:
+	case PIPE_CAPF_GUARD_BAND_BOTTOM:
+		return 0.0f;
+	}
+	return 0.0f;
+}
+
+static int r600_get_video_param(struct pipe_screen *screen,
+				enum pipe_video_profile profile,
+				enum pipe_video_entrypoint entrypoint,
+				enum pipe_video_cap param)
+{
+	switch (param) {
+	case PIPE_VIDEO_CAP_SUPPORTED:
+		return vl_profile_supported(screen, profile, entrypoint);
+	case PIPE_VIDEO_CAP_NPOT_TEXTURES:
+		return 1;
+	case PIPE_VIDEO_CAP_MAX_WIDTH:
+	case PIPE_VIDEO_CAP_MAX_HEIGHT:
+		return vl_video_buffer_max_size(screen);
+	case PIPE_VIDEO_CAP_PREFERED_FORMAT:
+		return PIPE_FORMAT_NV12;
+	case PIPE_VIDEO_CAP_PREFERS_INTERLACED:
+		return false;
+	case PIPE_VIDEO_CAP_SUPPORTS_INTERLACED:
+		return false;
+	case PIPE_VIDEO_CAP_SUPPORTS_PROGRESSIVE:
+		return true;
+	case PIPE_VIDEO_CAP_MAX_LEVEL:
+		return vl_level_supported(screen, profile);
+	default:
+		return 0;
+	}
+}
+
+const char *r600_get_llvm_processor_name(enum radeon_family family)
+{
+	switch (family) {
+	case CHIP_R600:
+	case CHIP_RV630:
+	case CHIP_RV635:
+	case CHIP_RV670:
+		return "r600";
+	case CHIP_RV610:
+	case CHIP_RV620:
+	case CHIP_RS780:
+	case CHIP_RS880:
+		return "rs880";
+	case CHIP_RV710:
+		return "rv710";
+	case CHIP_RV730:
+		return "rv730";
+	case CHIP_RV740:
+	case CHIP_RV770:
+		return "rv770";
+	case CHIP_PALM:
+	case CHIP_CEDAR:
+		return "cedar";
+	case CHIP_SUMO:
+	case CHIP_SUMO2:
+		return "sumo";
+	case CHIP_REDWOOD:
+		return "redwood";
+	case CHIP_JUNIPER:
+		return "juniper";
+	case CHIP_HEMLOCK:
+	case CHIP_CYPRESS:
+		return "cypress";
+	case CHIP_BARTS:
+		return "barts";
+	case CHIP_TURKS:
+		return "turks";
+	case CHIP_CAICOS:
+		return "caicos";
+	case CHIP_CAYMAN:
+        case CHIP_ARUBA:
+		return "cayman";
+
+	case CHIP_TAHITI: return "tahiti";
+	case CHIP_PITCAIRN: return "pitcairn";
+	case CHIP_VERDE: return "verde";
+	case CHIP_OLAND: return "oland";
+	case CHIP_HAINAN: return "hainan";
+	case CHIP_BONAIRE: return "bonaire";
+	case CHIP_KABINI: return "kabini";
+	case CHIP_KAVERI: return "kaveri";
+	case CHIP_HAWAII: return "hawaii";
+	case CHIP_MULLINS:
+		return "mullins";
+	case CHIP_TONGA: return "tonga";
+	case CHIP_ICELAND: return "iceland";
+	case CHIP_CARRIZO: return "carrizo";
+	case CHIP_FIJI:
+		return "fiji";
+	case CHIP_STONEY:
+		return "stoney";
+	case CHIP_POLARIS10:
+		return "polaris10";
+	case CHIP_POLARIS11:
+	case CHIP_POLARIS12: /* same as polaris11 */
+		return "polaris11";
+	case CHIP_VEGA10:
+	case CHIP_RAVEN:
+		return "gfx900";
+	default:
+		return "";
+	}
+}
+
+static unsigned get_max_threads_per_block(struct r600_common_screen *screen,
+					  enum pipe_shader_ir ir_type)
+{
+	if (ir_type != PIPE_SHADER_IR_TGSI)
+		return 256;
+
+	/* Only 16 waves per thread-group on gfx9. */
+	if (screen->chip_class >= GFX9)
+		return 1024;
+
+	/* Up to 40 waves per thread-group on GCN < gfx9. Expose a nice
+	 * round number.
+	 */
+	if (screen->chip_class >= SI)
+		return 2048;
+
+	return 256;
+}
+
+static int r600_get_compute_param(struct pipe_screen *screen,
+        enum pipe_shader_ir ir_type,
+        enum pipe_compute_cap param,
+        void *ret)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen *)screen;
+
+	//TODO: select these params by asic
+	switch (param) {
+	case PIPE_COMPUTE_CAP_IR_TARGET: {
+		const char *gpu;
+		const char *triple;
+		if (rscreen->family <= CHIP_ARUBA) {
+			triple = "r600--";
+		} else {
+			if (HAVE_LLVM < 0x0400) {
+				triple = "amdgcn--";
+			} else {
+				triple = "amdgcn-mesa-mesa3d";
+			}
+		}
+		switch(rscreen->family) {
+		/* Clang < 3.6 is missing Hainan in its list of
+		 * GPUs, so we need to use the name of a similar GPU.
+		 */
+		default:
+			gpu = r600_get_llvm_processor_name(rscreen->family);
+			break;
+		}
+		if (ret) {
+			sprintf(ret, "%s-%s", gpu, triple);
+		}
+		/* +2 for dash and terminating NIL byte */
+		return (strlen(triple) + strlen(gpu) + 2) * sizeof(char);
+	}
+	case PIPE_COMPUTE_CAP_GRID_DIMENSION:
+		if (ret) {
+			uint64_t *grid_dimension = ret;
+			grid_dimension[0] = 3;
+		}
+		return 1 * sizeof(uint64_t);
+
+	case PIPE_COMPUTE_CAP_MAX_GRID_SIZE:
+		if (ret) {
+			uint64_t *grid_size = ret;
+			grid_size[0] = 65535;
+			grid_size[1] = 65535;
+			grid_size[2] = 65535;
+		}
+		return 3 * sizeof(uint64_t) ;
+
+	case PIPE_COMPUTE_CAP_MAX_BLOCK_SIZE:
+		if (ret) {
+			uint64_t *block_size = ret;
+			unsigned threads_per_block = get_max_threads_per_block(rscreen, ir_type);
+			block_size[0] = threads_per_block;
+			block_size[1] = threads_per_block;
+			block_size[2] = threads_per_block;
+		}
+		return 3 * sizeof(uint64_t);
+
+	case PIPE_COMPUTE_CAP_MAX_THREADS_PER_BLOCK:
+		if (ret) {
+			uint64_t *max_threads_per_block = ret;
+			*max_threads_per_block = get_max_threads_per_block(rscreen, ir_type);
+		}
+		return sizeof(uint64_t);
+	case PIPE_COMPUTE_CAP_ADDRESS_BITS:
+		if (ret) {
+			uint32_t *address_bits = ret;
+			address_bits[0] = 32;
+			if (rscreen->chip_class >= SI)
+				address_bits[0] = 64;
+		}
+		return 1 * sizeof(uint32_t);
+
+	case PIPE_COMPUTE_CAP_MAX_GLOBAL_SIZE:
+		if (ret) {
+			uint64_t *max_global_size = ret;
+			uint64_t max_mem_alloc_size;
+
+			r600_get_compute_param(screen, ir_type,
+				PIPE_COMPUTE_CAP_MAX_MEM_ALLOC_SIZE,
+				&max_mem_alloc_size);
+
+			/* In OpenCL, the MAX_MEM_ALLOC_SIZE must be at least
+			 * 1/4 of the MAX_GLOBAL_SIZE.  Since the
+			 * MAX_MEM_ALLOC_SIZE is fixed for older kernels,
+			 * make sure we never report more than
+			 * 4 * MAX_MEM_ALLOC_SIZE.
+			 */
+			*max_global_size = MIN2(4 * max_mem_alloc_size,
+						MAX2(rscreen->info.gart_size,
+						     rscreen->info.vram_size));
+		}
+		return sizeof(uint64_t);
+
+	case PIPE_COMPUTE_CAP_MAX_LOCAL_SIZE:
+		if (ret) {
+			uint64_t *max_local_size = ret;
+			/* Value reported by the closed source driver. */
+			*max_local_size = 32768;
+		}
+		return sizeof(uint64_t);
+
+	case PIPE_COMPUTE_CAP_MAX_INPUT_SIZE:
+		if (ret) {
+			uint64_t *max_input_size = ret;
+			/* Value reported by the closed source driver. */
+			*max_input_size = 1024;
+		}
+		return sizeof(uint64_t);
+
+	case PIPE_COMPUTE_CAP_MAX_MEM_ALLOC_SIZE:
+		if (ret) {
+			uint64_t *max_mem_alloc_size = ret;
+
+			*max_mem_alloc_size = rscreen->info.max_alloc_size;
+		}
+		return sizeof(uint64_t);
+
+	case PIPE_COMPUTE_CAP_MAX_CLOCK_FREQUENCY:
+		if (ret) {
+			uint32_t *max_clock_frequency = ret;
+			*max_clock_frequency = rscreen->info.max_shader_clock;
+		}
+		return sizeof(uint32_t);
+
+	case PIPE_COMPUTE_CAP_MAX_COMPUTE_UNITS:
+		if (ret) {
+			uint32_t *max_compute_units = ret;
+			*max_compute_units = rscreen->info.num_good_compute_units;
+		}
+		return sizeof(uint32_t);
+
+	case PIPE_COMPUTE_CAP_IMAGES_SUPPORTED:
+		if (ret) {
+			uint32_t *images_supported = ret;
+			*images_supported = 0;
+		}
+		return sizeof(uint32_t);
+	case PIPE_COMPUTE_CAP_MAX_PRIVATE_SIZE:
+		break; /* unused */
+	case PIPE_COMPUTE_CAP_SUBGROUP_SIZE:
+		if (ret) {
+			uint32_t *subgroup_size = ret;
+			*subgroup_size = r600_wavefront_size(rscreen->family);
+		}
+		return sizeof(uint32_t);
+	case PIPE_COMPUTE_CAP_MAX_VARIABLE_THREADS_PER_BLOCK:
+		if (ret) {
+			uint64_t *max_variable_threads_per_block = ret;
+			if (rscreen->chip_class >= SI &&
+			    ir_type == PIPE_SHADER_IR_TGSI)
+				*max_variable_threads_per_block = SI_MAX_VARIABLE_THREADS_PER_BLOCK;
+			else
+				*max_variable_threads_per_block = 0;
+		}
+		return sizeof(uint64_t);
+	}
+
+        fprintf(stderr, "unknown PIPE_COMPUTE_CAP %d\n", param);
+        return 0;
+}
+
+static uint64_t r600_get_timestamp(struct pipe_screen *screen)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+
+	return 1000000 * rscreen->ws->query_value(rscreen->ws, RADEON_TIMESTAMP) /
+			rscreen->info.clock_crystal_freq;
+}
+
+static void r600_fence_reference(struct pipe_screen *screen,
+				 struct pipe_fence_handle **dst,
+				 struct pipe_fence_handle *src)
+{
+	struct radeon_winsys *ws = ((struct r600_common_screen*)screen)->ws;
+	struct r600_multi_fence **rdst = (struct r600_multi_fence **)dst;
+	struct r600_multi_fence *rsrc = (struct r600_multi_fence *)src;
+
+	if (pipe_reference(&(*rdst)->reference, &rsrc->reference)) {
+		ws->fence_reference(&(*rdst)->gfx, NULL);
+		ws->fence_reference(&(*rdst)->sdma, NULL);
+		FREE(*rdst);
+	}
+        *rdst = rsrc;
+}
+
+static boolean r600_fence_finish(struct pipe_screen *screen,
+				 struct pipe_context *ctx,
+				 struct pipe_fence_handle *fence,
+				 uint64_t timeout)
+{
+	struct radeon_winsys *rws = ((struct r600_common_screen*)screen)->ws;
+	struct r600_multi_fence *rfence = (struct r600_multi_fence *)fence;
+	struct r600_common_context *rctx;
+	int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
+
+	ctx = threaded_context_unwrap_sync(ctx);
+	rctx = ctx ? (struct r600_common_context*)ctx : NULL;
+
+	if (rfence->sdma) {
+		if (!rws->fence_wait(rws, rfence->sdma, timeout))
+			return false;
+
+		/* Recompute the timeout after waiting. */
+		if (timeout && timeout != PIPE_TIMEOUT_INFINITE) {
+			int64_t time = os_time_get_nano();
+			timeout = abs_timeout > time ? abs_timeout - time : 0;
+		}
+	}
+
+	if (!rfence->gfx)
+		return true;
+
+	/* Flush the gfx IB if it hasn't been flushed yet. */
+	if (rctx &&
+	    rfence->gfx_unflushed.ctx == rctx &&
+	    rfence->gfx_unflushed.ib_index == rctx->num_gfx_cs_flushes) {
+		rctx->gfx.flush(rctx, timeout ? 0 : RADEON_FLUSH_ASYNC, NULL);
+		rfence->gfx_unflushed.ctx = NULL;
+
+		if (!timeout)
+			return false;
+
+		/* Recompute the timeout after all that. */
+		if (timeout && timeout != PIPE_TIMEOUT_INFINITE) {
+			int64_t time = os_time_get_nano();
+			timeout = abs_timeout > time ? abs_timeout - time : 0;
+		}
+	}
+
+	return rws->fence_wait(rws, rfence->gfx, timeout);
+}
+
+static void r600_query_memory_info(struct pipe_screen *screen,
+				   struct pipe_memory_info *info)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+	struct radeon_winsys *ws = rscreen->ws;
+	unsigned vram_usage, gtt_usage;
+
+	info->total_device_memory = rscreen->info.vram_size / 1024;
+	info->total_staging_memory = rscreen->info.gart_size / 1024;
+
+	/* The real TTM memory usage is somewhat random, because:
+	 *
+	 * 1) TTM delays freeing memory, because it can only free it after
+	 *    fences expire.
+	 *
+	 * 2) The memory usage can be really low if big VRAM evictions are
+	 *    taking place, but the real usage is well above the size of VRAM.
+	 *
+	 * Instead, return statistics of this process.
+	 */
+	vram_usage = ws->query_value(ws, RADEON_REQUESTED_VRAM_MEMORY) / 1024;
+	gtt_usage =  ws->query_value(ws, RADEON_REQUESTED_GTT_MEMORY) / 1024;
+
+	info->avail_device_memory =
+		vram_usage <= info->total_device_memory ?
+				info->total_device_memory - vram_usage : 0;
+	info->avail_staging_memory =
+		gtt_usage <= info->total_staging_memory ?
+				info->total_staging_memory - gtt_usage : 0;
+
+	info->device_memory_evicted =
+		ws->query_value(ws, RADEON_NUM_BYTES_MOVED) / 1024;
+
+	if (rscreen->info.drm_major == 3 && rscreen->info.drm_minor >= 4)
+		info->nr_device_memory_evictions =
+			ws->query_value(ws, RADEON_NUM_EVICTIONS);
+	else
+		/* Just return the number of evicted 64KB pages. */
+		info->nr_device_memory_evictions = info->device_memory_evicted / 64;
+}
+
+struct pipe_resource *r600_resource_create_common(struct pipe_screen *screen,
+						  const struct pipe_resource *templ)
+{
+	if (templ->target == PIPE_BUFFER) {
+		return r600_buffer_create(screen, templ, 256);
+	} else {
+		return r600_texture_create(screen, templ);
+	}
+}
+
+bool r600_common_screen_init(struct r600_common_screen *rscreen,
+			     struct radeon_winsys *ws)
+{
+	char family_name[32] = {}, llvm_string[32] = {}, kernel_version[128] = {};
+	struct utsname uname_data;
+	const char *chip_name;
+
+	ws->query_info(ws, &rscreen->info);
+	rscreen->ws = ws;
+
+	if ((chip_name = r600_get_marketing_name(ws)))
+		snprintf(family_name, sizeof(family_name), "%s / ",
+			 r600_get_family_name(rscreen) + 4);
+	else
+		chip_name = r600_get_family_name(rscreen);
+
+	if (uname(&uname_data) == 0)
+		snprintf(kernel_version, sizeof(kernel_version),
+			 " / %s", uname_data.release);
+
+	if (HAVE_LLVM > 0) {
+		snprintf(llvm_string, sizeof(llvm_string),
+			 ", LLVM %i.%i.%i", (HAVE_LLVM >> 8) & 0xff,
+			 HAVE_LLVM & 0xff, MESA_LLVM_VERSION_PATCH);
+	}
+
+	snprintf(rscreen->renderer_string, sizeof(rscreen->renderer_string),
+		 "%s (%sDRM %i.%i.%i%s%s)",
+		 chip_name, family_name, rscreen->info.drm_major,
+		 rscreen->info.drm_minor, rscreen->info.drm_patchlevel,
+		 kernel_version, llvm_string);
+
+	rscreen->b.get_name = r600_get_name;
+	rscreen->b.get_vendor = r600_get_vendor;
+	rscreen->b.get_device_vendor = r600_get_device_vendor;
+	rscreen->b.get_disk_shader_cache = r600_get_disk_shader_cache;
+	rscreen->b.get_compute_param = r600_get_compute_param;
+	rscreen->b.get_paramf = r600_get_paramf;
+	rscreen->b.get_timestamp = r600_get_timestamp;
+	rscreen->b.fence_finish = r600_fence_finish;
+	rscreen->b.fence_reference = r600_fence_reference;
+	rscreen->b.resource_destroy = u_resource_destroy_vtbl;
+	rscreen->b.resource_from_user_memory = r600_buffer_from_user_memory;
+	rscreen->b.query_memory_info = r600_query_memory_info;
+
+	if (rscreen->info.has_hw_decode) {
+		rscreen->b.get_video_param = rvid_get_video_param;
+		rscreen->b.is_video_format_supported = rvid_is_format_supported;
+	} else {
+		rscreen->b.get_video_param = r600_get_video_param;
+		rscreen->b.is_video_format_supported = vl_video_buffer_is_format_supported;
+	}
+
+	r600_init_screen_texture_functions(rscreen);
+	r600_init_screen_query_functions(rscreen);
+
+	rscreen->family = rscreen->info.family;
+	rscreen->chip_class = rscreen->info.chip_class;
+	rscreen->debug_flags |= debug_get_flags_option("R600_DEBUG", common_debug_options, 0);
+	rscreen->has_rbplus = false;
+	rscreen->rbplus_allowed = false;
+
+	r600_disk_cache_create(rscreen);
+
+	slab_create_parent(&rscreen->pool_transfers, sizeof(struct r600_transfer), 64);
+
+	rscreen->force_aniso = MIN2(16, debug_get_num_option("R600_TEX_ANISO", -1));
+	if (rscreen->force_aniso >= 0) {
+		printf("radeon: Forcing anisotropy filter to %ix\n",
+		       /* round down to a power of two */
+		       1 << util_logbase2(rscreen->force_aniso));
+	}
+
+	util_format_s3tc_init();
+	(void) mtx_init(&rscreen->aux_context_lock, mtx_plain);
+	(void) mtx_init(&rscreen->gpu_load_mutex, mtx_plain);
+
+	if (rscreen->debug_flags & DBG_INFO) {
+		printf("pci (domain:bus:dev.func): %04x:%02x:%02x.%x\n",
+		       rscreen->info.pci_domain, rscreen->info.pci_bus,
+		       rscreen->info.pci_dev, rscreen->info.pci_func);
+		printf("pci_id = 0x%x\n", rscreen->info.pci_id);
+		printf("family = %i (%s)\n", rscreen->info.family,
+		       r600_get_family_name(rscreen));
+		printf("chip_class = %i\n", rscreen->info.chip_class);
+		printf("pte_fragment_size = %u\n", rscreen->info.pte_fragment_size);
+		printf("gart_page_size = %u\n", rscreen->info.gart_page_size);
+		printf("gart_size = %i MB\n", (int)DIV_ROUND_UP(rscreen->info.gart_size, 1024*1024));
+		printf("vram_size = %i MB\n", (int)DIV_ROUND_UP(rscreen->info.vram_size, 1024*1024));
+		printf("vram_vis_size = %i MB\n", (int)DIV_ROUND_UP(rscreen->info.vram_vis_size, 1024*1024));
+		printf("max_alloc_size = %i MB\n",
+		       (int)DIV_ROUND_UP(rscreen->info.max_alloc_size, 1024*1024));
+		printf("min_alloc_size = %u\n", rscreen->info.min_alloc_size);
+		printf("has_dedicated_vram = %u\n", rscreen->info.has_dedicated_vram);
+		printf("has_virtual_memory = %i\n", rscreen->info.has_virtual_memory);
+		printf("gfx_ib_pad_with_type2 = %i\n", rscreen->info.gfx_ib_pad_with_type2);
+		printf("has_hw_decode = %u\n", rscreen->info.has_hw_decode);
+		printf("num_sdma_rings = %i\n", rscreen->info.num_sdma_rings);
+		printf("num_compute_rings = %u\n", rscreen->info.num_compute_rings);
+		printf("uvd_fw_version = %u\n", rscreen->info.uvd_fw_version);
+		printf("vce_fw_version = %u\n", rscreen->info.vce_fw_version);
+		printf("me_fw_version = %i\n", rscreen->info.me_fw_version);
+		printf("pfp_fw_version = %i\n", rscreen->info.pfp_fw_version);
+		printf("ce_fw_version = %i\n", rscreen->info.ce_fw_version);
+		printf("vce_harvest_config = %i\n", rscreen->info.vce_harvest_config);
+		printf("clock_crystal_freq = %i\n", rscreen->info.clock_crystal_freq);
+		printf("tcc_cache_line_size = %u\n", rscreen->info.tcc_cache_line_size);
+		printf("drm = %i.%i.%i\n", rscreen->info.drm_major,
+		       rscreen->info.drm_minor, rscreen->info.drm_patchlevel);
+		printf("has_userptr = %i\n", rscreen->info.has_userptr);
+		printf("has_syncobj = %u\n", rscreen->info.has_syncobj);
+
+		printf("r600_max_quad_pipes = %i\n", rscreen->info.r600_max_quad_pipes);
+		printf("max_shader_clock = %i\n", rscreen->info.max_shader_clock);
+		printf("num_good_compute_units = %i\n", rscreen->info.num_good_compute_units);
+		printf("max_se = %i\n", rscreen->info.max_se);
+		printf("max_sh_per_se = %i\n", rscreen->info.max_sh_per_se);
+
+		printf("r600_gb_backend_map = %i\n", rscreen->info.r600_gb_backend_map);
+		printf("r600_gb_backend_map_valid = %i\n", rscreen->info.r600_gb_backend_map_valid);
+		printf("r600_num_banks = %i\n", rscreen->info.r600_num_banks);
+		printf("num_render_backends = %i\n", rscreen->info.num_render_backends);
+		printf("num_tile_pipes = %i\n", rscreen->info.num_tile_pipes);
+		printf("pipe_interleave_bytes = %i\n", rscreen->info.pipe_interleave_bytes);
+		printf("enabled_rb_mask = 0x%x\n", rscreen->info.enabled_rb_mask);
+		printf("max_alignment = %u\n", (unsigned)rscreen->info.max_alignment);
+	}
+	return true;
+}
+
+void r600_destroy_common_screen(struct r600_common_screen *rscreen)
+{
+	r600_perfcounters_destroy(rscreen);
+	r600_gpu_load_kill_thread(rscreen);
+
+	mtx_destroy(&rscreen->gpu_load_mutex);
+	mtx_destroy(&rscreen->aux_context_lock);
+	rscreen->aux_context->destroy(rscreen->aux_context);
+
+	slab_destroy_parent(&rscreen->pool_transfers);
+
+	disk_cache_destroy(rscreen->disk_shader_cache);
+	rscreen->ws->destroy(rscreen->ws);
+	FREE(rscreen);
+}
+
+bool r600_can_dump_shader(struct r600_common_screen *rscreen,
+			  unsigned processor)
+{
+	return rscreen->debug_flags & (1 << processor);
+}
+
+bool r600_extra_shader_checks(struct r600_common_screen *rscreen, unsigned processor)
+{
+	return (rscreen->debug_flags & DBG_CHECK_IR) ||
+	       r600_can_dump_shader(rscreen, processor);
+}
+
+void r600_screen_clear_buffer(struct r600_common_screen *rscreen, struct pipe_resource *dst,
+			      uint64_t offset, uint64_t size, unsigned value)
+{
+	struct r600_common_context *rctx = (struct r600_common_context*)rscreen->aux_context;
+
+	mtx_lock(&rscreen->aux_context_lock);
+	rctx->dma_clear_buffer(&rctx->b, dst, offset, size, value);
+	rscreen->aux_context->flush(rscreen->aux_context, NULL, 0);
+	mtx_unlock(&rscreen->aux_context_lock);
+}
diff --git a/src/gallium/drivers/r600/r600_pipe_common.h b/src/gallium/drivers/r600/r600_pipe_common.h
new file mode 100644
index 00000000000..39dd45aaceb
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_pipe_common.h
@@ -0,0 +1,1020 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors: Marek Olšák <[email protected]>
+ *
+ */
+
+/**
+ * This file contains common screen and context structures and functions
+ * for r600g and radeonsi.
+ */
+
+#ifndef R600_PIPE_COMMON_H
+#define R600_PIPE_COMMON_H
+
+#include <stdio.h>
+
+#include "amd/common/ac_binary.h"
+
+#include "radeon/radeon_winsys.h"
+
+#include "util/disk_cache.h"
+#include "util/u_blitter.h"
+#include "util/list.h"
+#include "util/u_range.h"
+#include "util/slab.h"
+#include "util/u_suballoc.h"
+#include "util/u_transfer.h"
+#include "util/u_threaded_context.h"
+
+struct u_log_context;
+
+#define ATI_VENDOR_ID 0x1002
+
+#define R600_RESOURCE_FLAG_TRANSFER		(PIPE_RESOURCE_FLAG_DRV_PRIV << 0)
+#define R600_RESOURCE_FLAG_FLUSHED_DEPTH	(PIPE_RESOURCE_FLAG_DRV_PRIV << 1)
+#define R600_RESOURCE_FLAG_FORCE_TILING		(PIPE_RESOURCE_FLAG_DRV_PRIV << 2)
+#define R600_RESOURCE_FLAG_DISABLE_DCC		(PIPE_RESOURCE_FLAG_DRV_PRIV << 3)
+#define R600_RESOURCE_FLAG_UNMAPPABLE		(PIPE_RESOURCE_FLAG_DRV_PRIV << 4)
+
+#define R600_CONTEXT_STREAMOUT_FLUSH		(1u << 0)
+/* Pipeline & streamout query controls. */
+#define R600_CONTEXT_START_PIPELINE_STATS	(1u << 1)
+#define R600_CONTEXT_STOP_PIPELINE_STATS	(1u << 2)
+#define R600_CONTEXT_FLUSH_FOR_RENDER_COND	(1u << 3)
+#define R600_CONTEXT_PRIVATE_FLAG		(1u << 4)
+
+/* special primitive types */
+#define R600_PRIM_RECTANGLE_LIST	PIPE_PRIM_MAX
+
+#define R600_NOT_QUERY		0xffffffff
+
+/* Debug flags. */
+#define DBG_VS			(1 << PIPE_SHADER_VERTEX)
+#define DBG_PS			(1 << PIPE_SHADER_FRAGMENT)
+#define DBG_GS			(1 << PIPE_SHADER_GEOMETRY)
+#define DBG_TCS			(1 << PIPE_SHADER_TESS_CTRL)
+#define DBG_TES			(1 << PIPE_SHADER_TESS_EVAL)
+#define DBG_CS			(1 << PIPE_SHADER_COMPUTE)
+#define DBG_ALL_SHADERS		(DBG_FS - 1)
+#define DBG_FS			(1 << 6) /* fetch shader */
+#define DBG_TEX			(1 << 7)
+#define DBG_NIR			(1 << 8)
+#define DBG_COMPUTE		(1 << 9)
+/* gap */
+#define DBG_VM			(1 << 11)
+#define DBG_NO_IR		(1 << 12)
+#define DBG_NO_TGSI		(1 << 13)
+#define DBG_NO_ASM		(1 << 14)
+#define DBG_PREOPT_IR		(1 << 15)
+#define DBG_CHECK_IR		(1 << 16)
+#define DBG_NO_OPT_VARIANT	(1 << 17)
+#define DBG_FS_CORRECT_DERIVS_AFTER_KILL (1 << 18)
+/* gaps */
+#define DBG_TEST_DMA		(1 << 20)
+/* Bits 21-31 are reserved for the r600g driver. */
+/* features */
+#define DBG_NO_ASYNC_DMA	(1ull << 32)
+#define DBG_NO_HYPERZ		(1ull << 33)
+#define DBG_NO_DISCARD_RANGE	(1ull << 34)
+#define DBG_NO_2D_TILING	(1ull << 35)
+#define DBG_NO_TILING		(1ull << 36)
+#define DBG_SWITCH_ON_EOP	(1ull << 37)
+#define DBG_FORCE_DMA		(1ull << 38)
+#define DBG_PRECOMPILE		(1ull << 39)
+#define DBG_INFO		(1ull << 40)
+#define DBG_NO_WC		(1ull << 41)
+#define DBG_CHECK_VM		(1ull << 42)
+#define DBG_NO_DCC		(1ull << 43)
+#define DBG_NO_DCC_CLEAR	(1ull << 44)
+#define DBG_NO_RB_PLUS		(1ull << 45)
+#define DBG_SI_SCHED		(1ull << 46)
+#define DBG_MONOLITHIC_SHADERS	(1ull << 47)
+/* gap */
+#define DBG_UNSAFE_MATH		(1ull << 49)
+#define DBG_NO_DCC_FB		(1ull << 50)
+#define DBG_TEST_VMFAULT_CP	(1ull << 51)
+#define DBG_TEST_VMFAULT_SDMA	(1ull << 52)
+#define DBG_TEST_VMFAULT_SHADER	(1ull << 53)
+#define DBG_NO_DPBB		(1ull << 54)
+#define DBG_NO_DFSM		(1ull << 55)
+
+#define R600_MAP_BUFFER_ALIGNMENT 64
+#define R600_MAX_VIEWPORTS        16
+
+#define SI_MAX_VARIABLE_THREADS_PER_BLOCK 1024
+
+enum r600_coherency {
+	R600_COHERENCY_NONE, /* no cache flushes needed */
+	R600_COHERENCY_SHADER,
+	R600_COHERENCY_CB_META,
+};
+
+#ifdef PIPE_ARCH_BIG_ENDIAN
+#define R600_BIG_ENDIAN 1
+#else
+#define R600_BIG_ENDIAN 0
+#endif
+
+struct r600_common_context;
+struct r600_perfcounters;
+struct tgsi_shader_info;
+struct r600_qbo_state;
+
+void radeon_shader_binary_init(struct ac_shader_binary *b);
+void radeon_shader_binary_clean(struct ac_shader_binary *b);
+
+/* Only 32-bit buffer allocations are supported, gallium doesn't support more
+ * at the moment.
+ */
+struct r600_resource {
+	struct threaded_resource	b;
+
+	/* Winsys objects. */
+	struct pb_buffer		*buf;
+	uint64_t			gpu_address;
+	/* Memory usage if the buffer placement is optimal. */
+	uint64_t			vram_usage;
+	uint64_t			gart_usage;
+
+	/* Resource properties. */
+	uint64_t			bo_size;
+	unsigned			bo_alignment;
+	enum radeon_bo_domain		domains;
+	enum radeon_bo_flag		flags;
+	unsigned			bind_history;
+
+	/* The buffer range which is initialized (with a write transfer,
+	 * streamout, DMA, or as a random access target). The rest of
+	 * the buffer is considered invalid and can be mapped unsynchronized.
+	 *
+	 * This allows unsychronized mapping of a buffer range which hasn't
+	 * been used yet. It's for applications which forget to use
+	 * the unsynchronized map flag and expect the driver to figure it out.
+         */
+	struct util_range		valid_buffer_range;
+
+	/* For buffers only. This indicates that a write operation has been
+	 * performed by TC L2, but the cache hasn't been flushed.
+	 * Any hw block which doesn't use or bypasses TC L2 should check this
+	 * flag and flush the cache before using the buffer.
+	 *
+	 * For example, TC L2 must be flushed if a buffer which has been
+	 * modified by a shader store instruction is about to be used as
+	 * an index buffer. The reason is that VGT DMA index fetching doesn't
+	 * use TC L2.
+	 */
+	bool				TC_L2_dirty;
+
+	/* Whether the resource has been exported via resource_get_handle. */
+	unsigned			external_usage; /* PIPE_HANDLE_USAGE_* */
+
+	/* Whether this resource is referenced by bindless handles. */
+	bool				texture_handle_allocated;
+	bool				image_handle_allocated;
+};
+
+struct r600_transfer {
+	struct threaded_transfer	b;
+	struct r600_resource		*staging;
+	unsigned			offset;
+};
+
+struct r600_fmask_info {
+	uint64_t offset;
+	uint64_t size;
+	unsigned alignment;
+	unsigned pitch_in_pixels;
+	unsigned bank_height;
+	unsigned slice_tile_max;
+	unsigned tile_mode_index;
+	unsigned tile_swizzle;
+};
+
+struct r600_cmask_info {
+	uint64_t offset;
+	uint64_t size;
+	unsigned alignment;
+	unsigned slice_tile_max;
+	uint64_t base_address_reg;
+};
+
+struct r600_texture {
+	struct r600_resource		resource;
+
+	uint64_t			size;
+	unsigned			num_level0_transfers;
+	enum pipe_format		db_render_format;
+	bool				is_depth;
+	bool				db_compatible;
+	bool				can_sample_z;
+	bool				can_sample_s;
+	unsigned			dirty_level_mask; /* each bit says if that mipmap is compressed */
+	unsigned			stencil_dirty_level_mask; /* each bit says if that mipmap is compressed */
+	struct r600_texture		*flushed_depth_texture;
+	struct radeon_surf		surface;
+
+	/* Colorbuffer compression and fast clear. */
+	struct r600_fmask_info		fmask;
+	struct r600_cmask_info		cmask;
+	struct r600_resource		*cmask_buffer;
+	uint64_t			dcc_offset; /* 0 = disabled */
+	unsigned			cb_color_info; /* fast clear enable bit */
+	unsigned			color_clear_value[2];
+	unsigned			last_msaa_resolve_target_micro_mode;
+
+	/* Depth buffer compression and fast clear. */
+	uint64_t			htile_offset;
+	bool				tc_compatible_htile;
+	bool				depth_cleared; /* if it was cleared at least once */
+	float				depth_clear_value;
+	bool				stencil_cleared; /* if it was cleared at least once */
+	uint8_t				stencil_clear_value;
+
+	bool				non_disp_tiling; /* R600-Cayman only */
+
+	/* Whether the texture is a displayable back buffer and needs DCC
+	 * decompression, which is expensive. Therefore, it's enabled only
+	 * if statistics suggest that it will pay off and it's allocated
+	 * separately. It can't be bound as a sampler by apps. Limited to
+	 * target == 2D and last_level == 0. If enabled, dcc_offset contains
+	 * the absolute GPUVM address, not the relative one.
+	 */
+	struct r600_resource		*dcc_separate_buffer;
+	/* When DCC is temporarily disabled, the separate buffer is here. */
+	struct r600_resource		*last_dcc_separate_buffer;
+	/* We need to track DCC dirtiness, because st/dri usually calls
+	 * flush_resource twice per frame (not a bug) and we don't wanna
+	 * decompress DCC twice. Also, the dirty tracking must be done even
+	 * if DCC isn't used, because it's required by the DCC usage analysis
+	 * for a possible future enablement.
+	 */
+	bool				separate_dcc_dirty;
+	/* Statistics gathering for the DCC enablement heuristic. */
+	bool				dcc_gather_statistics;
+	/* Estimate of how much this color buffer is written to in units of
+	 * full-screen draws: ps_invocations / (width * height)
+	 * Shader kills, late Z, and blending with trivial discards make it
+	 * inaccurate (we need to count CB updates, not PS invocations).
+	 */
+	unsigned			ps_draw_ratio;
+	/* The number of clears since the last DCC usage analysis. */
+	unsigned			num_slow_clears;
+
+	/* Counter that should be non-zero if the texture is bound to a
+	 * framebuffer. Implemented in radeonsi only.
+	 */
+	uint32_t			framebuffers_bound;
+};
+
+struct r600_surface {
+	struct pipe_surface		base;
+
+	/* These can vary with block-compressed textures. */
+	unsigned width0;
+	unsigned height0;
+
+	bool color_initialized;
+	bool depth_initialized;
+
+	/* Misc. color flags. */
+	bool alphatest_bypass;
+	bool export_16bpc;
+	bool color_is_int8;
+	bool color_is_int10;
+	bool dcc_incompatible;
+
+	/* Color registers. */
+	unsigned cb_color_info;
+	unsigned cb_color_base;
+	unsigned cb_color_view;
+	unsigned cb_color_size;		/* R600 only */
+	unsigned cb_color_dim;		/* EG only */
+	unsigned cb_color_pitch;	/* EG and later */
+	unsigned cb_color_slice;	/* EG and later */
+	unsigned cb_color_attrib;	/* EG and later */
+	unsigned cb_color_attrib2;	/* GFX9 and later */
+	unsigned cb_dcc_control;	/* VI and later */
+	unsigned cb_color_fmask;	/* CB_COLORn_FMASK (EG and later) or CB_COLORn_FRAG (r600) */
+	unsigned cb_color_fmask_slice;	/* EG and later */
+	unsigned cb_color_cmask;	/* CB_COLORn_TILE (r600 only) */
+	unsigned cb_color_mask;		/* R600 only */
+	unsigned spi_shader_col_format;		/* SI+, no blending, no alpha-to-coverage. */
+	unsigned spi_shader_col_format_alpha;	/* SI+, alpha-to-coverage */
+	unsigned spi_shader_col_format_blend;	/* SI+, blending without alpha. */
+	unsigned spi_shader_col_format_blend_alpha; /* SI+, blending with alpha. */
+	struct r600_resource *cb_buffer_fmask; /* Used for FMASK relocations. R600 only */
+	struct r600_resource *cb_buffer_cmask; /* Used for CMASK relocations. R600 only */
+
+	/* DB registers. */
+	uint64_t db_depth_base;		/* DB_Z_READ/WRITE_BASE (EG and later) or DB_DEPTH_BASE (r600) */
+	uint64_t db_stencil_base;	/* EG and later */
+	uint64_t db_htile_data_base;
+	unsigned db_depth_info;		/* R600 only, then SI and later */
+	unsigned db_z_info;		/* EG and later */
+	unsigned db_z_info2;		/* GFX9+ */
+	unsigned db_depth_view;
+	unsigned db_depth_size;
+	unsigned db_depth_slice;	/* EG and later */
+	unsigned db_stencil_info;	/* EG and later */
+	unsigned db_stencil_info2;	/* GFX9+ */
+	unsigned db_prefetch_limit;	/* R600 only */
+	unsigned db_htile_surface;
+	unsigned db_preload_control;	/* EG and later */
+};
+
+struct r600_mmio_counter {
+	unsigned busy;
+	unsigned idle;
+};
+
+union r600_mmio_counters {
+	struct {
+		/* For global GPU load including SDMA. */
+		struct r600_mmio_counter gpu;
+
+		/* GRBM_STATUS */
+		struct r600_mmio_counter spi;
+		struct r600_mmio_counter gui;
+		struct r600_mmio_counter ta;
+		struct r600_mmio_counter gds;
+		struct r600_mmio_counter vgt;
+		struct r600_mmio_counter ia;
+		struct r600_mmio_counter sx;
+		struct r600_mmio_counter wd;
+		struct r600_mmio_counter bci;
+		struct r600_mmio_counter sc;
+		struct r600_mmio_counter pa;
+		struct r600_mmio_counter db;
+		struct r600_mmio_counter cp;
+		struct r600_mmio_counter cb;
+
+		/* SRBM_STATUS2 */
+		struct r600_mmio_counter sdma;
+
+		/* CP_STAT */
+		struct r600_mmio_counter pfp;
+		struct r600_mmio_counter meq;
+		struct r600_mmio_counter me;
+		struct r600_mmio_counter surf_sync;
+		struct r600_mmio_counter cp_dma;
+		struct r600_mmio_counter scratch_ram;
+	} named;
+	unsigned array[0];
+};
+
+struct r600_memory_object {
+	struct pipe_memory_object	b;
+	struct pb_buffer		*buf;
+	uint32_t			stride;
+	uint32_t			offset;
+};
+
+struct r600_common_screen {
+	struct pipe_screen		b;
+	struct radeon_winsys		*ws;
+	enum radeon_family		family;
+	enum chip_class			chip_class;
+	struct radeon_info		info;
+	uint64_t			debug_flags;
+	bool				has_cp_dma;
+	bool				has_streamout;
+	bool				has_rbplus;     /* if RB+ registers exist */
+	bool				rbplus_allowed; /* if RB+ is allowed */
+
+	struct disk_cache		*disk_shader_cache;
+
+	struct slab_parent_pool		pool_transfers;
+
+	/* Texture filter settings. */
+	int				force_aniso; /* -1 = disabled */
+
+	/* Auxiliary context. Mainly used to initialize resources.
+	 * It must be locked prior to using and flushed before unlocking. */
+	struct pipe_context		*aux_context;
+	mtx_t				aux_context_lock;
+
+	/* This must be in the screen, because UE4 uses one context for
+	 * compilation and another one for rendering.
+	 */
+	unsigned			num_compilations;
+	/* Along with ST_DEBUG=precompile, this should show if applications
+	 * are loading shaders on demand. This is a monotonic counter.
+	 */
+	unsigned			num_shaders_created;
+	unsigned			num_shader_cache_hits;
+
+	/* GPU load thread. */
+	mtx_t				gpu_load_mutex;
+	thrd_t				gpu_load_thread;
+	union r600_mmio_counters	mmio_counters;
+	volatile unsigned		gpu_load_stop_thread; /* bool */
+
+	char				renderer_string[100];
+
+	/* Performance counters. */
+	struct r600_perfcounters	*perfcounters;
+
+	/* If pipe_screen wants to recompute and re-emit the framebuffer,
+	 * sampler, and image states of all contexts, it should atomically
+	 * increment this.
+	 *
+	 * Each context will compare this with its own last known value of
+	 * the counter before drawing and re-emit the states accordingly.
+	 */
+	unsigned			dirty_tex_counter;
+
+	/* Atomically increment this counter when an existing texture's
+	 * metadata is enabled or disabled in a way that requires changing
+	 * contexts' compressed texture binding masks.
+	 */
+	unsigned			compressed_colortex_counter;
+
+	struct {
+		/* Context flags to set so that all writes from earlier jobs
+		 * in the CP are seen by L2 clients.
+		 */
+		unsigned cp_to_L2;
+
+		/* Context flags to set so that all writes from earlier jobs
+		 * that end in L2 are seen by CP.
+		 */
+		unsigned L2_to_cp;
+
+		/* Context flags to set so that all writes from earlier
+		 * compute jobs are seen by L2 clients.
+		 */
+		unsigned compute_to_L2;
+	} barrier_flags;
+
+	void (*query_opaque_metadata)(struct r600_common_screen *rscreen,
+				      struct r600_texture *rtex,
+				      struct radeon_bo_metadata *md);
+
+	void (*apply_opaque_metadata)(struct r600_common_screen *rscreen,
+				    struct r600_texture *rtex,
+				    struct radeon_bo_metadata *md);
+};
+
+/* This encapsulates a state or an operation which can emitted into the GPU
+ * command stream. */
+struct r600_atom {
+	void (*emit)(struct r600_common_context *ctx, struct r600_atom *state);
+	unsigned		num_dw;
+	unsigned short		id;
+};
+
+struct r600_so_target {
+	struct pipe_stream_output_target b;
+
+	/* The buffer where BUFFER_FILLED_SIZE is stored. */
+	struct r600_resource	*buf_filled_size;
+	unsigned		buf_filled_size_offset;
+	bool			buf_filled_size_valid;
+
+	unsigned		stride_in_dw;
+};
+
+struct r600_streamout {
+	struct r600_atom		begin_atom;
+	bool				begin_emitted;
+	unsigned			num_dw_for_end;
+
+	unsigned			enabled_mask;
+	unsigned			num_targets;
+	struct r600_so_target		*targets[PIPE_MAX_SO_BUFFERS];
+
+	unsigned			append_bitmask;
+	bool				suspended;
+
+	/* External state which comes from the vertex shader,
+	 * it must be set explicitly when binding a shader. */
+	uint16_t			*stride_in_dw;
+	unsigned			enabled_stream_buffers_mask; /* stream0 buffers0-3 in 4 LSB */
+
+	/* The state of VGT_STRMOUT_BUFFER_(CONFIG|EN). */
+	unsigned			hw_enabled_mask;
+
+	/* The state of VGT_STRMOUT_(CONFIG|EN). */
+	struct r600_atom		enable_atom;
+	bool				streamout_enabled;
+	bool				prims_gen_query_enabled;
+	int				num_prims_gen_queries;
+};
+
+struct r600_signed_scissor {
+	int minx;
+	int miny;
+	int maxx;
+	int maxy;
+};
+
+struct r600_scissors {
+	struct r600_atom		atom;
+	unsigned			dirty_mask;
+	struct pipe_scissor_state	states[R600_MAX_VIEWPORTS];
+};
+
+struct r600_viewports {
+	struct r600_atom		atom;
+	unsigned			dirty_mask;
+	unsigned			depth_range_dirty_mask;
+	struct pipe_viewport_state	states[R600_MAX_VIEWPORTS];
+	struct r600_signed_scissor	as_scissor[R600_MAX_VIEWPORTS];
+};
+
+struct r600_ring {
+	struct radeon_winsys_cs		*cs;
+	void (*flush)(void *ctx, unsigned flags,
+		      struct pipe_fence_handle **fence);
+};
+
+/* Saved CS data for debugging features. */
+struct radeon_saved_cs {
+	uint32_t			*ib;
+	unsigned			num_dw;
+
+	struct radeon_bo_list_item	*bo_list;
+	unsigned			bo_count;
+};
+
+struct r600_common_context {
+	struct pipe_context b; /* base class */
+
+	struct r600_common_screen	*screen;
+	struct radeon_winsys		*ws;
+	struct radeon_winsys_ctx	*ctx;
+	enum radeon_family		family;
+	enum chip_class			chip_class;
+	struct r600_ring		gfx;
+	struct r600_ring		dma;
+	struct pipe_fence_handle	*last_gfx_fence;
+	struct pipe_fence_handle	*last_sdma_fence;
+	struct r600_resource		*eop_bug_scratch;
+	unsigned			num_gfx_cs_flushes;
+	unsigned			initial_gfx_cs_size;
+	unsigned			gpu_reset_counter;
+	unsigned			last_dirty_tex_counter;
+	unsigned			last_compressed_colortex_counter;
+	unsigned			last_num_draw_calls;
+
+	struct threaded_context		*tc;
+	struct u_suballocator		*allocator_zeroed_memory;
+	struct slab_child_pool		pool_transfers;
+	struct slab_child_pool		pool_transfers_unsync; /* for threaded_context */
+
+	/* Current unaccounted memory usage. */
+	uint64_t			vram;
+	uint64_t			gtt;
+
+	/* States. */
+	struct r600_streamout		streamout;
+	struct r600_scissors		scissors;
+	struct r600_viewports		viewports;
+	bool				scissor_enabled;
+	bool				clip_halfz;
+	bool				vs_writes_viewport_index;
+	bool				vs_disables_clipping_viewport;
+
+	/* Additional context states. */
+	unsigned flags; /* flush flags */
+
+	/* Queries. */
+	/* Maintain the list of active queries for pausing between IBs. */
+	int				num_occlusion_queries;
+	int				num_perfect_occlusion_queries;
+	struct list_head		active_queries;
+	unsigned			num_cs_dw_queries_suspend;
+	/* Misc stats. */
+	unsigned			num_draw_calls;
+	unsigned			num_decompress_calls;
+	unsigned			num_mrt_draw_calls;
+	unsigned			num_prim_restart_calls;
+	unsigned			num_spill_draw_calls;
+	unsigned			num_compute_calls;
+	unsigned			num_spill_compute_calls;
+	unsigned			num_dma_calls;
+	unsigned			num_cp_dma_calls;
+	unsigned			num_vs_flushes;
+	unsigned			num_ps_flushes;
+	unsigned			num_cs_flushes;
+	unsigned			num_cb_cache_flushes;
+	unsigned			num_db_cache_flushes;
+	unsigned			num_L2_invalidates;
+	unsigned			num_L2_writebacks;
+	unsigned			num_resident_handles;
+	uint64_t			num_alloc_tex_transfer_bytes;
+	unsigned			last_tex_ps_draw_ratio; /* for query */
+
+	/* Render condition. */
+	struct r600_atom		render_cond_atom;
+	struct pipe_query		*render_cond;
+	unsigned			render_cond_mode;
+	bool				render_cond_invert;
+	bool				render_cond_force_off; /* for u_blitter */
+
+	/* MSAA sample locations.
+	 * The first index is the sample index.
+	 * The second index is the coordinate: X, Y. */
+	float				sample_locations_1x[1][2];
+	float				sample_locations_2x[2][2];
+	float				sample_locations_4x[4][2];
+	float				sample_locations_8x[8][2];
+	float				sample_locations_16x[16][2];
+
+	/* Statistics gathering for the DCC enablement heuristic. It can't be
+	 * in r600_texture because r600_texture can be shared by multiple
+	 * contexts. This is for back buffers only. We shouldn't get too many
+	 * of those.
+	 *
+	 * X11 DRI3 rotates among a finite set of back buffers. They should
+	 * all fit in this array. If they don't, separate DCC might never be
+	 * enabled by DCC stat gathering.
+	 */
+	struct {
+		struct r600_texture		*tex;
+		/* Query queue: 0 = usually active, 1 = waiting, 2 = readback. */
+		struct pipe_query		*ps_stats[3];
+		/* If all slots are used and another slot is needed,
+		 * the least recently used slot is evicted based on this. */
+		int64_t				last_use_timestamp;
+		bool				query_active;
+	} dcc_stats[5];
+
+	struct pipe_debug_callback	debug;
+	struct pipe_device_reset_callback device_reset_callback;
+	struct u_log_context		*log;
+
+	void				*query_result_shader;
+
+	/* Copy one resource to another using async DMA. */
+	void (*dma_copy)(struct pipe_context *ctx,
+			 struct pipe_resource *dst,
+			 unsigned dst_level,
+			 unsigned dst_x, unsigned dst_y, unsigned dst_z,
+			 struct pipe_resource *src,
+			 unsigned src_level,
+			 const struct pipe_box *src_box);
+
+	void (*dma_clear_buffer)(struct pipe_context *ctx, struct pipe_resource *dst,
+				 uint64_t offset, uint64_t size, unsigned value);
+
+	void (*clear_buffer)(struct pipe_context *ctx, struct pipe_resource *dst,
+			     uint64_t offset, uint64_t size, unsigned value,
+			     enum r600_coherency coher);
+
+	void (*blit_decompress_depth)(struct pipe_context *ctx,
+				      struct r600_texture *texture,
+				      struct r600_texture *staging,
+				      unsigned first_level, unsigned last_level,
+				      unsigned first_layer, unsigned last_layer,
+				      unsigned first_sample, unsigned last_sample);
+
+	void (*decompress_dcc)(struct pipe_context *ctx,
+			       struct r600_texture *rtex);
+
+	/* Reallocate the buffer and update all resource bindings where
+	 * the buffer is bound, including all resource descriptors. */
+	void (*invalidate_buffer)(struct pipe_context *ctx, struct pipe_resource *buf);
+
+	/* Update all resource bindings where the buffer is bound, including
+	 * all resource descriptors. This is invalidate_buffer without
+	 * the invalidation. */
+	void (*rebind_buffer)(struct pipe_context *ctx, struct pipe_resource *buf,
+			      uint64_t old_gpu_address);
+
+	/* Enable or disable occlusion queries. */
+	void (*set_occlusion_query_state)(struct pipe_context *ctx, bool enable);
+
+	void (*save_qbo_state)(struct pipe_context *ctx, struct r600_qbo_state *st);
+
+	/* This ensures there is enough space in the command stream. */
+	void (*need_gfx_cs_space)(struct pipe_context *ctx, unsigned num_dw,
+				  bool include_draw_vbo);
+
+	void (*set_atom_dirty)(struct r600_common_context *ctx,
+			       struct r600_atom *atom, bool dirty);
+
+	void (*check_vm_faults)(struct r600_common_context *ctx,
+				struct radeon_saved_cs *saved,
+				enum ring_type ring);
+};
+
+/* r600_buffer_common.c */
+bool r600_rings_is_buffer_referenced(struct r600_common_context *ctx,
+				     struct pb_buffer *buf,
+				     enum radeon_bo_usage usage);
+void *r600_buffer_map_sync_with_rings(struct r600_common_context *ctx,
+                                      struct r600_resource *resource,
+                                      unsigned usage);
+void r600_buffer_subdata(struct pipe_context *ctx,
+			 struct pipe_resource *buffer,
+			 unsigned usage, unsigned offset,
+			 unsigned size, const void *data);
+void r600_init_resource_fields(struct r600_common_screen *rscreen,
+			       struct r600_resource *res,
+			       uint64_t size, unsigned alignment);
+bool r600_alloc_resource(struct r600_common_screen *rscreen,
+			 struct r600_resource *res);
+struct pipe_resource *r600_buffer_create(struct pipe_screen *screen,
+					 const struct pipe_resource *templ,
+					 unsigned alignment);
+struct pipe_resource * r600_aligned_buffer_create(struct pipe_screen *screen,
+						  unsigned flags,
+						  unsigned usage,
+						  unsigned size,
+						  unsigned alignment);
+struct pipe_resource *
+r600_buffer_from_user_memory(struct pipe_screen *screen,
+			     const struct pipe_resource *templ,
+			     void *user_memory);
+void
+r600_invalidate_resource(struct pipe_context *ctx,
+			 struct pipe_resource *resource);
+void r600_replace_buffer_storage(struct pipe_context *ctx,
+				 struct pipe_resource *dst,
+				 struct pipe_resource *src);
+
+/* r600_common_pipe.c */
+void r600_gfx_write_event_eop(struct r600_common_context *ctx,
+			      unsigned event, unsigned event_flags,
+			      unsigned data_sel,
+			      struct r600_resource *buf, uint64_t va,
+			      uint32_t new_fence, unsigned query_type);
+unsigned r600_gfx_write_fence_dwords(struct r600_common_screen *screen);
+void r600_gfx_wait_fence(struct r600_common_context *ctx,
+			 uint64_t va, uint32_t ref, uint32_t mask);
+void r600_draw_rectangle(struct blitter_context *blitter,
+			 int x1, int y1, int x2, int y2,
+			 float depth, unsigned num_instances,
+			 enum blitter_attrib_type type,
+			 const union blitter_attrib *attrib);
+bool r600_common_screen_init(struct r600_common_screen *rscreen,
+			     struct radeon_winsys *ws);
+void r600_destroy_common_screen(struct r600_common_screen *rscreen);
+void r600_preflush_suspend_features(struct r600_common_context *ctx);
+void r600_postflush_resume_features(struct r600_common_context *ctx);
+bool r600_common_context_init(struct r600_common_context *rctx,
+			      struct r600_common_screen *rscreen,
+			      unsigned context_flags);
+void r600_common_context_cleanup(struct r600_common_context *rctx);
+bool r600_can_dump_shader(struct r600_common_screen *rscreen,
+			  unsigned processor);
+bool r600_extra_shader_checks(struct r600_common_screen *rscreen,
+			      unsigned processor);
+void r600_screen_clear_buffer(struct r600_common_screen *rscreen, struct pipe_resource *dst,
+			      uint64_t offset, uint64_t size, unsigned value);
+struct pipe_resource *r600_resource_create_common(struct pipe_screen *screen,
+						  const struct pipe_resource *templ);
+const char *r600_get_llvm_processor_name(enum radeon_family family);
+void r600_need_dma_space(struct r600_common_context *ctx, unsigned num_dw,
+			 struct r600_resource *dst, struct r600_resource *src);
+void radeon_save_cs(struct radeon_winsys *ws, struct radeon_winsys_cs *cs,
+		    struct radeon_saved_cs *saved, bool get_buffer_list);
+void radeon_clear_saved_cs(struct radeon_saved_cs *saved);
+bool r600_check_device_reset(struct r600_common_context *rctx);
+
+/* r600_gpu_load.c */
+void r600_gpu_load_kill_thread(struct r600_common_screen *rscreen);
+uint64_t r600_begin_counter(struct r600_common_screen *rscreen, unsigned type);
+unsigned r600_end_counter(struct r600_common_screen *rscreen, unsigned type,
+			  uint64_t begin);
+
+/* r600_perfcounters.c */
+void r600_perfcounters_destroy(struct r600_common_screen *rscreen);
+
+/* r600_query.c */
+void r600_init_screen_query_functions(struct r600_common_screen *rscreen);
+void r600_query_init(struct r600_common_context *rctx);
+void r600_suspend_queries(struct r600_common_context *ctx);
+void r600_resume_queries(struct r600_common_context *ctx);
+void r600_query_fix_enabled_rb_mask(struct r600_common_screen *rscreen);
+
+/* r600_streamout.c */
+void r600_streamout_buffers_dirty(struct r600_common_context *rctx);
+void r600_set_streamout_targets(struct pipe_context *ctx,
+				unsigned num_targets,
+				struct pipe_stream_output_target **targets,
+				const unsigned *offset);
+void r600_emit_streamout_end(struct r600_common_context *rctx);
+void r600_update_prims_generated_query_state(struct r600_common_context *rctx,
+					     unsigned type, int diff);
+void r600_streamout_init(struct r600_common_context *rctx);
+
+/* r600_test_dma.c */
+void r600_test_dma(struct r600_common_screen *rscreen);
+
+/* r600_texture.c */
+bool r600_prepare_for_dma_blit(struct r600_common_context *rctx,
+				struct r600_texture *rdst,
+				unsigned dst_level, unsigned dstx,
+				unsigned dsty, unsigned dstz,
+				struct r600_texture *rsrc,
+				unsigned src_level,
+				const struct pipe_box *src_box);
+void r600_texture_get_fmask_info(struct r600_common_screen *rscreen,
+				 struct r600_texture *rtex,
+				 unsigned nr_samples,
+				 struct r600_fmask_info *out);
+void r600_texture_get_cmask_info(struct r600_common_screen *rscreen,
+				 struct r600_texture *rtex,
+				 struct r600_cmask_info *out);
+bool r600_init_flushed_depth_texture(struct pipe_context *ctx,
+				     struct pipe_resource *texture,
+				     struct r600_texture **staging);
+void r600_print_texture_info(struct r600_common_screen *rscreen,
+			     struct r600_texture *rtex, struct u_log_context *log);
+struct pipe_resource *r600_texture_create(struct pipe_screen *screen,
+					const struct pipe_resource *templ);
+struct pipe_surface *r600_create_surface_custom(struct pipe_context *pipe,
+						struct pipe_resource *texture,
+						const struct pipe_surface *templ,
+						unsigned width0, unsigned height0,
+						unsigned width, unsigned height);
+unsigned r600_translate_colorswap(enum pipe_format format, bool do_endian_swap);
+void evergreen_do_fast_color_clear(struct r600_common_context *rctx,
+				   struct pipe_framebuffer_state *fb,
+				   struct r600_atom *fb_state,
+				   unsigned *buffers, ubyte *dirty_cbufs,
+				   const union pipe_color_union *color);
+bool r600_texture_disable_dcc(struct r600_common_context *rctx,
+			      struct r600_texture *rtex);
+void r600_init_screen_texture_functions(struct r600_common_screen *rscreen);
+void r600_init_context_texture_functions(struct r600_common_context *rctx);
+
+/* r600_viewport.c */
+void evergreen_apply_scissor_bug_workaround(struct r600_common_context *rctx,
+					    struct pipe_scissor_state *scissor);
+void r600_viewport_set_rast_deps(struct r600_common_context *rctx,
+				 bool scissor_enable, bool clip_halfz);
+void r600_update_vs_writes_viewport_index(struct r600_common_context *rctx,
+					  struct tgsi_shader_info *info);
+void r600_init_viewport_functions(struct r600_common_context *rctx);
+
+/* cayman_msaa.c */
+extern const uint32_t eg_sample_locs_2x[4];
+extern const unsigned eg_max_dist_2x;
+extern const uint32_t eg_sample_locs_4x[4];
+extern const unsigned eg_max_dist_4x;
+void cayman_get_sample_position(struct pipe_context *ctx, unsigned sample_count,
+				unsigned sample_index, float *out_value);
+void cayman_init_msaa(struct pipe_context *ctx);
+void cayman_emit_msaa_sample_locs(struct radeon_winsys_cs *cs, int nr_samples);
+void cayman_emit_msaa_config(struct radeon_winsys_cs *cs, int nr_samples,
+			     int ps_iter_samples, int overrast_samples,
+			     unsigned sc_mode_cntl_1);
+
+
+/* Inline helpers. */
+
+static inline struct r600_resource *r600_resource(struct pipe_resource *r)
+{
+	return (struct r600_resource*)r;
+}
+
+static inline void
+r600_resource_reference(struct r600_resource **ptr, struct r600_resource *res)
+{
+	pipe_resource_reference((struct pipe_resource **)ptr,
+				(struct pipe_resource *)res);
+}
+
+static inline void
+r600_texture_reference(struct r600_texture **ptr, struct r600_texture *res)
+{
+	pipe_resource_reference((struct pipe_resource **)ptr, &res->resource.b.b);
+}
+
+static inline void
+r600_context_add_resource_size(struct pipe_context *ctx, struct pipe_resource *r)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct r600_resource *res = (struct r600_resource *)r;
+
+	if (res) {
+		/* Add memory usage for need_gfx_cs_space */
+		rctx->vram += res->vram_usage;
+		rctx->gtt += res->gart_usage;
+	}
+}
+
+static inline bool r600_get_strmout_en(struct r600_common_context *rctx)
+{
+	return rctx->streamout.streamout_enabled ||
+	       rctx->streamout.prims_gen_query_enabled;
+}
+
+#define     SQ_TEX_XY_FILTER_POINT                         0x00
+#define     SQ_TEX_XY_FILTER_BILINEAR                      0x01
+#define     SQ_TEX_XY_FILTER_ANISO_POINT                   0x02
+#define     SQ_TEX_XY_FILTER_ANISO_BILINEAR                0x03
+
+static inline unsigned eg_tex_filter(unsigned filter, unsigned max_aniso)
+{
+	if (filter == PIPE_TEX_FILTER_LINEAR)
+		return max_aniso > 1 ? SQ_TEX_XY_FILTER_ANISO_BILINEAR
+				     : SQ_TEX_XY_FILTER_BILINEAR;
+	else
+		return max_aniso > 1 ? SQ_TEX_XY_FILTER_ANISO_POINT
+				     : SQ_TEX_XY_FILTER_POINT;
+}
+
+static inline unsigned r600_tex_aniso_filter(unsigned filter)
+{
+	if (filter < 2)
+		return 0;
+	if (filter < 4)
+		return 1;
+	if (filter < 8)
+		return 2;
+	if (filter < 16)
+		return 3;
+	return 4;
+}
+
+static inline unsigned r600_wavefront_size(enum radeon_family family)
+{
+	switch (family) {
+	case CHIP_RV610:
+	case CHIP_RS780:
+	case CHIP_RV620:
+	case CHIP_RS880:
+		return 16;
+	case CHIP_RV630:
+	case CHIP_RV635:
+	case CHIP_RV730:
+	case CHIP_RV710:
+	case CHIP_PALM:
+	case CHIP_CEDAR:
+		return 32;
+	default:
+		return 64;
+	}
+}
+
+static inline enum radeon_bo_priority
+r600_get_sampler_view_priority(struct r600_resource *res)
+{
+	if (res->b.b.target == PIPE_BUFFER)
+		return RADEON_PRIO_SAMPLER_BUFFER;
+
+	if (res->b.b.nr_samples > 1)
+		return RADEON_PRIO_SAMPLER_TEXTURE_MSAA;
+
+	return RADEON_PRIO_SAMPLER_TEXTURE;
+}
+
+static inline bool
+r600_can_sample_zs(struct r600_texture *tex, bool stencil_sampler)
+{
+	return (stencil_sampler && tex->can_sample_s) ||
+	       (!stencil_sampler && tex->can_sample_z);
+}
+
+static inline bool
+r600_htile_enabled(struct r600_texture *tex, unsigned level)
+{
+	return tex->htile_offset && level == 0;
+}
+
+#define COMPUTE_DBG(rscreen, fmt, args...) \
+	do { \
+		if ((rscreen->b.debug_flags & DBG_COMPUTE)) fprintf(stderr, fmt, ##args); \
+	} while (0);
+
+#define R600_ERR(fmt, args...) \
+	fprintf(stderr, "EE %s:%d %s - " fmt, __FILE__, __LINE__, __func__, ##args)
+
+/* For MSAA sample positions. */
+#define FILL_SREG(s0x, s0y, s1x, s1y, s2x, s2y, s3x, s3y)  \
+	(((s0x) & 0xf) | (((unsigned)(s0y) & 0xf) << 4) |		   \
+	(((unsigned)(s1x) & 0xf) << 8) | (((unsigned)(s1y) & 0xf) << 12) |	   \
+	(((unsigned)(s2x) & 0xf) << 16) | (((unsigned)(s2y) & 0xf) << 20) |	   \
+	 (((unsigned)(s3x) & 0xf) << 24) | (((unsigned)(s3y) & 0xf) << 28))
+
+static inline int S_FIXED(float value, unsigned frac_bits)
+{
+	return value * (1 << frac_bits);
+}
+
+#endif
diff --git a/src/gallium/drivers/r600/r600_query.c b/src/gallium/drivers/r600/r600_query.c
new file mode 100644
index 00000000000..03ff1018a71
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_query.c
@@ -0,0 +1,2201 @@
+/*
+ * Copyright 2010 Jerome Glisse <[email protected]>
+ * Copyright 2014 Marek Olšák <[email protected]>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "r600_query.h"
+#include "r600_cs.h"
+#include "util/u_memory.h"
+#include "util/u_upload_mgr.h"
+#include "os/os_time.h"
+#include "tgsi/tgsi_text.h"
+
+#define R600_MAX_STREAMS 4
+
+struct r600_hw_query_params {
+	unsigned start_offset;
+	unsigned end_offset;
+	unsigned fence_offset;
+	unsigned pair_stride;
+	unsigned pair_count;
+};
+
+/* Queries without buffer handling or suspend/resume. */
+struct r600_query_sw {
+	struct r600_query b;
+
+	uint64_t begin_result;
+	uint64_t end_result;
+
+	uint64_t begin_time;
+	uint64_t end_time;
+
+	/* Fence for GPU_FINISHED. */
+	struct pipe_fence_handle *fence;
+};
+
+static void r600_query_sw_destroy(struct r600_common_screen *rscreen,
+				  struct r600_query *rquery)
+{
+	struct r600_query_sw *query = (struct r600_query_sw *)rquery;
+
+	rscreen->b.fence_reference(&rscreen->b, &query->fence, NULL);
+	FREE(query);
+}
+
+static enum radeon_value_id winsys_id_from_type(unsigned type)
+{
+	switch (type) {
+	case R600_QUERY_REQUESTED_VRAM: return RADEON_REQUESTED_VRAM_MEMORY;
+	case R600_QUERY_REQUESTED_GTT: return RADEON_REQUESTED_GTT_MEMORY;
+	case R600_QUERY_MAPPED_VRAM: return RADEON_MAPPED_VRAM;
+	case R600_QUERY_MAPPED_GTT: return RADEON_MAPPED_GTT;
+	case R600_QUERY_BUFFER_WAIT_TIME: return RADEON_BUFFER_WAIT_TIME_NS;
+	case R600_QUERY_NUM_MAPPED_BUFFERS: return RADEON_NUM_MAPPED_BUFFERS;
+	case R600_QUERY_NUM_GFX_IBS: return RADEON_NUM_GFX_IBS;
+	case R600_QUERY_NUM_SDMA_IBS: return RADEON_NUM_SDMA_IBS;
+	case R600_QUERY_GFX_BO_LIST_SIZE: return RADEON_GFX_BO_LIST_COUNTER;
+	case R600_QUERY_NUM_BYTES_MOVED: return RADEON_NUM_BYTES_MOVED;
+	case R600_QUERY_NUM_EVICTIONS: return RADEON_NUM_EVICTIONS;
+	case R600_QUERY_NUM_VRAM_CPU_PAGE_FAULTS: return RADEON_NUM_VRAM_CPU_PAGE_FAULTS;
+	case R600_QUERY_VRAM_USAGE: return RADEON_VRAM_USAGE;
+	case R600_QUERY_VRAM_VIS_USAGE: return RADEON_VRAM_VIS_USAGE;
+	case R600_QUERY_GTT_USAGE: return RADEON_GTT_USAGE;
+	case R600_QUERY_GPU_TEMPERATURE: return RADEON_GPU_TEMPERATURE;
+	case R600_QUERY_CURRENT_GPU_SCLK: return RADEON_CURRENT_SCLK;
+	case R600_QUERY_CURRENT_GPU_MCLK: return RADEON_CURRENT_MCLK;
+	case R600_QUERY_CS_THREAD_BUSY: return RADEON_CS_THREAD_TIME;
+	default: unreachable("query type does not correspond to winsys id");
+	}
+}
+
+static bool r600_query_sw_begin(struct r600_common_context *rctx,
+				struct r600_query *rquery)
+{
+	struct r600_query_sw *query = (struct r600_query_sw *)rquery;
+	enum radeon_value_id ws_id;
+
+	switch(query->b.type) {
+	case PIPE_QUERY_TIMESTAMP_DISJOINT:
+	case PIPE_QUERY_GPU_FINISHED:
+		break;
+	case R600_QUERY_DRAW_CALLS:
+		query->begin_result = rctx->num_draw_calls;
+		break;
+	case R600_QUERY_DECOMPRESS_CALLS:
+		query->begin_result = rctx->num_decompress_calls;
+		break;
+	case R600_QUERY_MRT_DRAW_CALLS:
+		query->begin_result = rctx->num_mrt_draw_calls;
+		break;
+	case R600_QUERY_PRIM_RESTART_CALLS:
+		query->begin_result = rctx->num_prim_restart_calls;
+		break;
+	case R600_QUERY_SPILL_DRAW_CALLS:
+		query->begin_result = rctx->num_spill_draw_calls;
+		break;
+	case R600_QUERY_COMPUTE_CALLS:
+		query->begin_result = rctx->num_compute_calls;
+		break;
+	case R600_QUERY_SPILL_COMPUTE_CALLS:
+		query->begin_result = rctx->num_spill_compute_calls;
+		break;
+	case R600_QUERY_DMA_CALLS:
+		query->begin_result = rctx->num_dma_calls;
+		break;
+	case R600_QUERY_CP_DMA_CALLS:
+		query->begin_result = rctx->num_cp_dma_calls;
+		break;
+	case R600_QUERY_NUM_VS_FLUSHES:
+		query->begin_result = rctx->num_vs_flushes;
+		break;
+	case R600_QUERY_NUM_PS_FLUSHES:
+		query->begin_result = rctx->num_ps_flushes;
+		break;
+	case R600_QUERY_NUM_CS_FLUSHES:
+		query->begin_result = rctx->num_cs_flushes;
+		break;
+	case R600_QUERY_NUM_CB_CACHE_FLUSHES:
+		query->begin_result = rctx->num_cb_cache_flushes;
+		break;
+	case R600_QUERY_NUM_DB_CACHE_FLUSHES:
+		query->begin_result = rctx->num_db_cache_flushes;
+		break;
+	case R600_QUERY_NUM_L2_INVALIDATES:
+		query->begin_result = rctx->num_L2_invalidates;
+		break;
+	case R600_QUERY_NUM_L2_WRITEBACKS:
+		query->begin_result = rctx->num_L2_writebacks;
+		break;
+	case R600_QUERY_NUM_RESIDENT_HANDLES:
+		query->begin_result = rctx->num_resident_handles;
+		break;
+	case R600_QUERY_TC_OFFLOADED_SLOTS:
+		query->begin_result = rctx->tc ? rctx->tc->num_offloaded_slots : 0;
+		break;
+	case R600_QUERY_TC_DIRECT_SLOTS:
+		query->begin_result = rctx->tc ? rctx->tc->num_direct_slots : 0;
+		break;
+	case R600_QUERY_TC_NUM_SYNCS:
+		query->begin_result = rctx->tc ? rctx->tc->num_syncs : 0;
+		break;
+	case R600_QUERY_REQUESTED_VRAM:
+	case R600_QUERY_REQUESTED_GTT:
+	case R600_QUERY_MAPPED_VRAM:
+	case R600_QUERY_MAPPED_GTT:
+	case R600_QUERY_VRAM_USAGE:
+	case R600_QUERY_VRAM_VIS_USAGE:
+	case R600_QUERY_GTT_USAGE:
+	case R600_QUERY_GPU_TEMPERATURE:
+	case R600_QUERY_CURRENT_GPU_SCLK:
+	case R600_QUERY_CURRENT_GPU_MCLK:
+	case R600_QUERY_BACK_BUFFER_PS_DRAW_RATIO:
+	case R600_QUERY_NUM_MAPPED_BUFFERS:
+		query->begin_result = 0;
+		break;
+	case R600_QUERY_BUFFER_WAIT_TIME:
+	case R600_QUERY_NUM_GFX_IBS:
+	case R600_QUERY_NUM_SDMA_IBS:
+	case R600_QUERY_NUM_BYTES_MOVED:
+	case R600_QUERY_NUM_EVICTIONS:
+	case R600_QUERY_NUM_VRAM_CPU_PAGE_FAULTS: {
+		enum radeon_value_id ws_id = winsys_id_from_type(query->b.type);
+		query->begin_result = rctx->ws->query_value(rctx->ws, ws_id);
+		break;
+	}
+	case R600_QUERY_GFX_BO_LIST_SIZE:
+		ws_id = winsys_id_from_type(query->b.type);
+		query->begin_result = rctx->ws->query_value(rctx->ws, ws_id);
+		query->begin_time = rctx->ws->query_value(rctx->ws,
+							  RADEON_NUM_GFX_IBS);
+		break;
+	case R600_QUERY_CS_THREAD_BUSY:
+		ws_id = winsys_id_from_type(query->b.type);
+		query->begin_result = rctx->ws->query_value(rctx->ws, ws_id);
+		query->begin_time = os_time_get_nano();
+		break;
+	case R600_QUERY_GALLIUM_THREAD_BUSY:
+		query->begin_result =
+			rctx->tc ? util_queue_get_thread_time_nano(&rctx->tc->queue, 0) : 0;
+		query->begin_time = os_time_get_nano();
+		break;
+	case R600_QUERY_GPU_LOAD:
+	case R600_QUERY_GPU_SHADERS_BUSY:
+	case R600_QUERY_GPU_TA_BUSY:
+	case R600_QUERY_GPU_GDS_BUSY:
+	case R600_QUERY_GPU_VGT_BUSY:
+	case R600_QUERY_GPU_IA_BUSY:
+	case R600_QUERY_GPU_SX_BUSY:
+	case R600_QUERY_GPU_WD_BUSY:
+	case R600_QUERY_GPU_BCI_BUSY:
+	case R600_QUERY_GPU_SC_BUSY:
+	case R600_QUERY_GPU_PA_BUSY:
+	case R600_QUERY_GPU_DB_BUSY:
+	case R600_QUERY_GPU_CP_BUSY:
+	case R600_QUERY_GPU_CB_BUSY:
+	case R600_QUERY_GPU_SDMA_BUSY:
+	case R600_QUERY_GPU_PFP_BUSY:
+	case R600_QUERY_GPU_MEQ_BUSY:
+	case R600_QUERY_GPU_ME_BUSY:
+	case R600_QUERY_GPU_SURF_SYNC_BUSY:
+	case R600_QUERY_GPU_CP_DMA_BUSY:
+	case R600_QUERY_GPU_SCRATCH_RAM_BUSY:
+		query->begin_result = r600_begin_counter(rctx->screen,
+							 query->b.type);
+		break;
+	case R600_QUERY_NUM_COMPILATIONS:
+		query->begin_result = p_atomic_read(&rctx->screen->num_compilations);
+		break;
+	case R600_QUERY_NUM_SHADERS_CREATED:
+		query->begin_result = p_atomic_read(&rctx->screen->num_shaders_created);
+		break;
+	case R600_QUERY_NUM_SHADER_CACHE_HITS:
+		query->begin_result =
+			p_atomic_read(&rctx->screen->num_shader_cache_hits);
+		break;
+	case R600_QUERY_GPIN_ASIC_ID:
+	case R600_QUERY_GPIN_NUM_SIMD:
+	case R600_QUERY_GPIN_NUM_RB:
+	case R600_QUERY_GPIN_NUM_SPI:
+	case R600_QUERY_GPIN_NUM_SE:
+		break;
+	default:
+		unreachable("r600_query_sw_begin: bad query type");
+	}
+
+	return true;
+}
+
+static bool r600_query_sw_end(struct r600_common_context *rctx,
+			      struct r600_query *rquery)
+{
+	struct r600_query_sw *query = (struct r600_query_sw *)rquery;
+	enum radeon_value_id ws_id;
+
+	switch(query->b.type) {
+	case PIPE_QUERY_TIMESTAMP_DISJOINT:
+		break;
+	case PIPE_QUERY_GPU_FINISHED:
+		rctx->b.flush(&rctx->b, &query->fence, PIPE_FLUSH_DEFERRED);
+		break;
+	case R600_QUERY_DRAW_CALLS:
+		query->end_result = rctx->num_draw_calls;
+		break;
+	case R600_QUERY_DECOMPRESS_CALLS:
+		query->end_result = rctx->num_decompress_calls;
+		break;
+	case R600_QUERY_MRT_DRAW_CALLS:
+		query->end_result = rctx->num_mrt_draw_calls;
+		break;
+	case R600_QUERY_PRIM_RESTART_CALLS:
+		query->end_result = rctx->num_prim_restart_calls;
+		break;
+	case R600_QUERY_SPILL_DRAW_CALLS:
+		query->end_result = rctx->num_spill_draw_calls;
+		break;
+	case R600_QUERY_COMPUTE_CALLS:
+		query->end_result = rctx->num_compute_calls;
+		break;
+	case R600_QUERY_SPILL_COMPUTE_CALLS:
+		query->end_result = rctx->num_spill_compute_calls;
+		break;
+	case R600_QUERY_DMA_CALLS:
+		query->end_result = rctx->num_dma_calls;
+		break;
+	case R600_QUERY_CP_DMA_CALLS:
+		query->end_result = rctx->num_cp_dma_calls;
+		break;
+	case R600_QUERY_NUM_VS_FLUSHES:
+		query->end_result = rctx->num_vs_flushes;
+		break;
+	case R600_QUERY_NUM_PS_FLUSHES:
+		query->end_result = rctx->num_ps_flushes;
+		break;
+	case R600_QUERY_NUM_CS_FLUSHES:
+		query->end_result = rctx->num_cs_flushes;
+		break;
+	case R600_QUERY_NUM_CB_CACHE_FLUSHES:
+		query->end_result = rctx->num_cb_cache_flushes;
+		break;
+	case R600_QUERY_NUM_DB_CACHE_FLUSHES:
+		query->end_result = rctx->num_db_cache_flushes;
+		break;
+	case R600_QUERY_NUM_L2_INVALIDATES:
+		query->end_result = rctx->num_L2_invalidates;
+		break;
+	case R600_QUERY_NUM_L2_WRITEBACKS:
+		query->end_result = rctx->num_L2_writebacks;
+		break;
+	case R600_QUERY_NUM_RESIDENT_HANDLES:
+		query->end_result = rctx->num_resident_handles;
+		break;
+	case R600_QUERY_TC_OFFLOADED_SLOTS:
+		query->end_result = rctx->tc ? rctx->tc->num_offloaded_slots : 0;
+		break;
+	case R600_QUERY_TC_DIRECT_SLOTS:
+		query->end_result = rctx->tc ? rctx->tc->num_direct_slots : 0;
+		break;
+	case R600_QUERY_TC_NUM_SYNCS:
+		query->end_result = rctx->tc ? rctx->tc->num_syncs : 0;
+		break;
+	case R600_QUERY_REQUESTED_VRAM:
+	case R600_QUERY_REQUESTED_GTT:
+	case R600_QUERY_MAPPED_VRAM:
+	case R600_QUERY_MAPPED_GTT:
+	case R600_QUERY_VRAM_USAGE:
+	case R600_QUERY_VRAM_VIS_USAGE:
+	case R600_QUERY_GTT_USAGE:
+	case R600_QUERY_GPU_TEMPERATURE:
+	case R600_QUERY_CURRENT_GPU_SCLK:
+	case R600_QUERY_CURRENT_GPU_MCLK:
+	case R600_QUERY_BUFFER_WAIT_TIME:
+	case R600_QUERY_NUM_MAPPED_BUFFERS:
+	case R600_QUERY_NUM_GFX_IBS:
+	case R600_QUERY_NUM_SDMA_IBS:
+	case R600_QUERY_NUM_BYTES_MOVED:
+	case R600_QUERY_NUM_EVICTIONS:
+	case R600_QUERY_NUM_VRAM_CPU_PAGE_FAULTS: {
+		enum radeon_value_id ws_id = winsys_id_from_type(query->b.type);
+		query->end_result = rctx->ws->query_value(rctx->ws, ws_id);
+		break;
+	}
+	case R600_QUERY_GFX_BO_LIST_SIZE:
+		ws_id = winsys_id_from_type(query->b.type);
+		query->end_result = rctx->ws->query_value(rctx->ws, ws_id);
+		query->end_time = rctx->ws->query_value(rctx->ws,
+							RADEON_NUM_GFX_IBS);
+		break;
+	case R600_QUERY_CS_THREAD_BUSY:
+		ws_id = winsys_id_from_type(query->b.type);
+		query->end_result = rctx->ws->query_value(rctx->ws, ws_id);
+		query->end_time = os_time_get_nano();
+		break;
+	case R600_QUERY_GALLIUM_THREAD_BUSY:
+		query->end_result =
+			rctx->tc ? util_queue_get_thread_time_nano(&rctx->tc->queue, 0) : 0;
+		query->end_time = os_time_get_nano();
+		break;
+	case R600_QUERY_GPU_LOAD:
+	case R600_QUERY_GPU_SHADERS_BUSY:
+	case R600_QUERY_GPU_TA_BUSY:
+	case R600_QUERY_GPU_GDS_BUSY:
+	case R600_QUERY_GPU_VGT_BUSY:
+	case R600_QUERY_GPU_IA_BUSY:
+	case R600_QUERY_GPU_SX_BUSY:
+	case R600_QUERY_GPU_WD_BUSY:
+	case R600_QUERY_GPU_BCI_BUSY:
+	case R600_QUERY_GPU_SC_BUSY:
+	case R600_QUERY_GPU_PA_BUSY:
+	case R600_QUERY_GPU_DB_BUSY:
+	case R600_QUERY_GPU_CP_BUSY:
+	case R600_QUERY_GPU_CB_BUSY:
+	case R600_QUERY_GPU_SDMA_BUSY:
+	case R600_QUERY_GPU_PFP_BUSY:
+	case R600_QUERY_GPU_MEQ_BUSY:
+	case R600_QUERY_GPU_ME_BUSY:
+	case R600_QUERY_GPU_SURF_SYNC_BUSY:
+	case R600_QUERY_GPU_CP_DMA_BUSY:
+	case R600_QUERY_GPU_SCRATCH_RAM_BUSY:
+		query->end_result = r600_end_counter(rctx->screen,
+						     query->b.type,
+						     query->begin_result);
+		query->begin_result = 0;
+		break;
+	case R600_QUERY_NUM_COMPILATIONS:
+		query->end_result = p_atomic_read(&rctx->screen->num_compilations);
+		break;
+	case R600_QUERY_NUM_SHADERS_CREATED:
+		query->end_result = p_atomic_read(&rctx->screen->num_shaders_created);
+		break;
+	case R600_QUERY_BACK_BUFFER_PS_DRAW_RATIO:
+		query->end_result = rctx->last_tex_ps_draw_ratio;
+		break;
+	case R600_QUERY_NUM_SHADER_CACHE_HITS:
+		query->end_result =
+			p_atomic_read(&rctx->screen->num_shader_cache_hits);
+		break;
+	case R600_QUERY_GPIN_ASIC_ID:
+	case R600_QUERY_GPIN_NUM_SIMD:
+	case R600_QUERY_GPIN_NUM_RB:
+	case R600_QUERY_GPIN_NUM_SPI:
+	case R600_QUERY_GPIN_NUM_SE:
+		break;
+	default:
+		unreachable("r600_query_sw_end: bad query type");
+	}
+
+	return true;
+}
+
+static bool r600_query_sw_get_result(struct r600_common_context *rctx,
+				     struct r600_query *rquery,
+				     bool wait,
+				     union pipe_query_result *result)
+{
+	struct r600_query_sw *query = (struct r600_query_sw *)rquery;
+
+	switch (query->b.type) {
+	case PIPE_QUERY_TIMESTAMP_DISJOINT:
+		/* Convert from cycles per millisecond to cycles per second (Hz). */
+		result->timestamp_disjoint.frequency =
+			(uint64_t)rctx->screen->info.clock_crystal_freq * 1000;
+		result->timestamp_disjoint.disjoint = false;
+		return true;
+	case PIPE_QUERY_GPU_FINISHED: {
+		struct pipe_screen *screen = rctx->b.screen;
+		struct pipe_context *ctx = rquery->b.flushed ? NULL : &rctx->b;
+
+		result->b = screen->fence_finish(screen, ctx, query->fence,
+						 wait ? PIPE_TIMEOUT_INFINITE : 0);
+		return result->b;
+	}
+
+	case R600_QUERY_GFX_BO_LIST_SIZE:
+		result->u64 = (query->end_result - query->begin_result) /
+			      (query->end_time - query->begin_time);
+		return true;
+	case R600_QUERY_CS_THREAD_BUSY:
+	case R600_QUERY_GALLIUM_THREAD_BUSY:
+		result->u64 = (query->end_result - query->begin_result) * 100 /
+			      (query->end_time - query->begin_time);
+		return true;
+	case R600_QUERY_GPIN_ASIC_ID:
+		result->u32 = 0;
+		return true;
+	case R600_QUERY_GPIN_NUM_SIMD:
+		result->u32 = rctx->screen->info.num_good_compute_units;
+		return true;
+	case R600_QUERY_GPIN_NUM_RB:
+		result->u32 = rctx->screen->info.num_render_backends;
+		return true;
+	case R600_QUERY_GPIN_NUM_SPI:
+		result->u32 = 1; /* all supported chips have one SPI per SE */
+		return true;
+	case R600_QUERY_GPIN_NUM_SE:
+		result->u32 = rctx->screen->info.max_se;
+		return true;
+	}
+
+	result->u64 = query->end_result - query->begin_result;
+
+	switch (query->b.type) {
+	case R600_QUERY_BUFFER_WAIT_TIME:
+	case R600_QUERY_GPU_TEMPERATURE:
+		result->u64 /= 1000;
+		break;
+	case R600_QUERY_CURRENT_GPU_SCLK:
+	case R600_QUERY_CURRENT_GPU_MCLK:
+		result->u64 *= 1000000;
+		break;
+	}
+
+	return true;
+}
+
+
+static struct r600_query_ops sw_query_ops = {
+	.destroy = r600_query_sw_destroy,
+	.begin = r600_query_sw_begin,
+	.end = r600_query_sw_end,
+	.get_result = r600_query_sw_get_result,
+	.get_result_resource = NULL
+};
+
+static struct pipe_query *r600_query_sw_create(unsigned query_type)
+{
+	struct r600_query_sw *query;
+
+	query = CALLOC_STRUCT(r600_query_sw);
+	if (!query)
+		return NULL;
+
+	query->b.type = query_type;
+	query->b.ops = &sw_query_ops;
+
+	return (struct pipe_query *)query;
+}
+
+void r600_query_hw_destroy(struct r600_common_screen *rscreen,
+			   struct r600_query *rquery)
+{
+	struct r600_query_hw *query = (struct r600_query_hw *)rquery;
+	struct r600_query_buffer *prev = query->buffer.previous;
+
+	/* Release all query buffers. */
+	while (prev) {
+		struct r600_query_buffer *qbuf = prev;
+		prev = prev->previous;
+		r600_resource_reference(&qbuf->buf, NULL);
+		FREE(qbuf);
+	}
+
+	r600_resource_reference(&query->buffer.buf, NULL);
+	r600_resource_reference(&query->workaround_buf, NULL);
+	FREE(rquery);
+}
+
+static struct r600_resource *r600_new_query_buffer(struct r600_common_screen *rscreen,
+						   struct r600_query_hw *query)
+{
+	unsigned buf_size = MAX2(query->result_size,
+				 rscreen->info.min_alloc_size);
+
+	/* Queries are normally read by the CPU after
+	 * being written by the gpu, hence staging is probably a good
+	 * usage pattern.
+	 */
+	struct r600_resource *buf = (struct r600_resource*)
+		pipe_buffer_create(&rscreen->b, 0,
+				   PIPE_USAGE_STAGING, buf_size);
+	if (!buf)
+		return NULL;
+
+	if (!query->ops->prepare_buffer(rscreen, query, buf)) {
+		r600_resource_reference(&buf, NULL);
+		return NULL;
+	}
+
+	return buf;
+}
+
+static bool r600_query_hw_prepare_buffer(struct r600_common_screen *rscreen,
+					 struct r600_query_hw *query,
+					 struct r600_resource *buffer)
+{
+	/* Callers ensure that the buffer is currently unused by the GPU. */
+	uint32_t *results = rscreen->ws->buffer_map(buffer->buf, NULL,
+						   PIPE_TRANSFER_WRITE |
+						   PIPE_TRANSFER_UNSYNCHRONIZED);
+	if (!results)
+		return false;
+
+	memset(results, 0, buffer->b.b.width0);
+
+	if (query->b.type == PIPE_QUERY_OCCLUSION_COUNTER ||
+	    query->b.type == PIPE_QUERY_OCCLUSION_PREDICATE) {
+		unsigned max_rbs = rscreen->info.num_render_backends;
+		unsigned enabled_rb_mask = rscreen->info.enabled_rb_mask;
+		unsigned num_results;
+		unsigned i, j;
+
+		/* Set top bits for unused backends. */
+		num_results = buffer->b.b.width0 / query->result_size;
+		for (j = 0; j < num_results; j++) {
+			for (i = 0; i < max_rbs; i++) {
+				if (!(enabled_rb_mask & (1<<i))) {
+					results[(i * 4)+1] = 0x80000000;
+					results[(i * 4)+3] = 0x80000000;
+				}
+			}
+			results += 4 * max_rbs;
+		}
+	}
+
+	return true;
+}
+
+static void r600_query_hw_get_result_resource(struct r600_common_context *rctx,
+                                              struct r600_query *rquery,
+                                              bool wait,
+                                              enum pipe_query_value_type result_type,
+                                              int index,
+                                              struct pipe_resource *resource,
+                                              unsigned offset);
+
+static struct r600_query_ops query_hw_ops = {
+	.destroy = r600_query_hw_destroy,
+	.begin = r600_query_hw_begin,
+	.end = r600_query_hw_end,
+	.get_result = r600_query_hw_get_result,
+	.get_result_resource = r600_query_hw_get_result_resource,
+};
+
+static void r600_query_hw_do_emit_start(struct r600_common_context *ctx,
+					struct r600_query_hw *query,
+					struct r600_resource *buffer,
+					uint64_t va);
+static void r600_query_hw_do_emit_stop(struct r600_common_context *ctx,
+				       struct r600_query_hw *query,
+				       struct r600_resource *buffer,
+				       uint64_t va);
+static void r600_query_hw_add_result(struct r600_common_screen *rscreen,
+				     struct r600_query_hw *, void *buffer,
+				     union pipe_query_result *result);
+static void r600_query_hw_clear_result(struct r600_query_hw *,
+				       union pipe_query_result *);
+
+static struct r600_query_hw_ops query_hw_default_hw_ops = {
+	.prepare_buffer = r600_query_hw_prepare_buffer,
+	.emit_start = r600_query_hw_do_emit_start,
+	.emit_stop = r600_query_hw_do_emit_stop,
+	.clear_result = r600_query_hw_clear_result,
+	.add_result = r600_query_hw_add_result,
+};
+
+bool r600_query_hw_init(struct r600_common_screen *rscreen,
+			struct r600_query_hw *query)
+{
+	query->buffer.buf = r600_new_query_buffer(rscreen, query);
+	if (!query->buffer.buf)
+		return false;
+
+	return true;
+}
+
+static struct pipe_query *r600_query_hw_create(struct r600_common_screen *rscreen,
+					       unsigned query_type,
+					       unsigned index)
+{
+	struct r600_query_hw *query = CALLOC_STRUCT(r600_query_hw);
+	if (!query)
+		return NULL;
+
+	query->b.type = query_type;
+	query->b.ops = &query_hw_ops;
+	query->ops = &query_hw_default_hw_ops;
+
+	switch (query_type) {
+	case PIPE_QUERY_OCCLUSION_COUNTER:
+	case PIPE_QUERY_OCCLUSION_PREDICATE:
+		query->result_size = 16 * rscreen->info.num_render_backends;
+		query->result_size += 16; /* for the fence + alignment */
+		query->num_cs_dw_begin = 6;
+		query->num_cs_dw_end = 6 + r600_gfx_write_fence_dwords(rscreen);
+		break;
+	case PIPE_QUERY_TIME_ELAPSED:
+		query->result_size = 24;
+		query->num_cs_dw_begin = 8;
+		query->num_cs_dw_end = 8 + r600_gfx_write_fence_dwords(rscreen);
+		break;
+	case PIPE_QUERY_TIMESTAMP:
+		query->result_size = 16;
+		query->num_cs_dw_end = 8 + r600_gfx_write_fence_dwords(rscreen);
+		query->flags = R600_QUERY_HW_FLAG_NO_START;
+		break;
+	case PIPE_QUERY_PRIMITIVES_EMITTED:
+	case PIPE_QUERY_PRIMITIVES_GENERATED:
+	case PIPE_QUERY_SO_STATISTICS:
+	case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
+		/* NumPrimitivesWritten, PrimitiveStorageNeeded. */
+		query->result_size = 32;
+		query->num_cs_dw_begin = 6;
+		query->num_cs_dw_end = 6;
+		query->stream = index;
+		break;
+	case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
+		/* NumPrimitivesWritten, PrimitiveStorageNeeded. */
+		query->result_size = 32 * R600_MAX_STREAMS;
+		query->num_cs_dw_begin = 6 * R600_MAX_STREAMS;
+		query->num_cs_dw_end = 6 * R600_MAX_STREAMS;
+		break;
+	case PIPE_QUERY_PIPELINE_STATISTICS:
+		/* 11 values on EG, 8 on R600. */
+		query->result_size = (rscreen->chip_class >= EVERGREEN ? 11 : 8) * 16;
+		query->result_size += 8; /* for the fence + alignment */
+		query->num_cs_dw_begin = 6;
+		query->num_cs_dw_end = 6 + r600_gfx_write_fence_dwords(rscreen);
+		break;
+	default:
+		assert(0);
+		FREE(query);
+		return NULL;
+	}
+
+	if (!r600_query_hw_init(rscreen, query)) {
+		FREE(query);
+		return NULL;
+	}
+
+	return (struct pipe_query *)query;
+}
+
+static void r600_update_occlusion_query_state(struct r600_common_context *rctx,
+					      unsigned type, int diff)
+{
+	if (type == PIPE_QUERY_OCCLUSION_COUNTER ||
+	    type == PIPE_QUERY_OCCLUSION_PREDICATE) {
+		bool old_enable = rctx->num_occlusion_queries != 0;
+		bool old_perfect_enable =
+			rctx->num_perfect_occlusion_queries != 0;
+		bool enable, perfect_enable;
+
+		rctx->num_occlusion_queries += diff;
+		assert(rctx->num_occlusion_queries >= 0);
+
+		if (type == PIPE_QUERY_OCCLUSION_COUNTER) {
+			rctx->num_perfect_occlusion_queries += diff;
+			assert(rctx->num_perfect_occlusion_queries >= 0);
+		}
+
+		enable = rctx->num_occlusion_queries != 0;
+		perfect_enable = rctx->num_perfect_occlusion_queries != 0;
+
+		if (enable != old_enable || perfect_enable != old_perfect_enable) {
+			rctx->set_occlusion_query_state(&rctx->b, enable);
+		}
+	}
+}
+
+static unsigned event_type_for_stream(unsigned stream)
+{
+	switch (stream) {
+	default:
+	case 0: return EVENT_TYPE_SAMPLE_STREAMOUTSTATS;
+	case 1: return EVENT_TYPE_SAMPLE_STREAMOUTSTATS1;
+	case 2: return EVENT_TYPE_SAMPLE_STREAMOUTSTATS2;
+	case 3: return EVENT_TYPE_SAMPLE_STREAMOUTSTATS3;
+	}
+}
+
+static void emit_sample_streamout(struct radeon_winsys_cs *cs, uint64_t va,
+				  unsigned stream)
+{
+	radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
+	radeon_emit(cs, EVENT_TYPE(event_type_for_stream(stream)) | EVENT_INDEX(3));
+	radeon_emit(cs, va);
+	radeon_emit(cs, va >> 32);
+}
+
+static void r600_query_hw_do_emit_start(struct r600_common_context *ctx,
+					struct r600_query_hw *query,
+					struct r600_resource *buffer,
+					uint64_t va)
+{
+	struct radeon_winsys_cs *cs = ctx->gfx.cs;
+
+	switch (query->b.type) {
+	case PIPE_QUERY_OCCLUSION_COUNTER:
+	case PIPE_QUERY_OCCLUSION_PREDICATE:
+		radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
+		radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1));
+		radeon_emit(cs, va);
+		radeon_emit(cs, va >> 32);
+		break;
+	case PIPE_QUERY_PRIMITIVES_EMITTED:
+	case PIPE_QUERY_PRIMITIVES_GENERATED:
+	case PIPE_QUERY_SO_STATISTICS:
+	case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
+		emit_sample_streamout(cs, va, query->stream);
+		break;
+	case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
+		for (unsigned stream = 0; stream < R600_MAX_STREAMS; ++stream)
+			emit_sample_streamout(cs, va + 32 * stream, stream);
+		break;
+	case PIPE_QUERY_TIME_ELAPSED:
+		if (ctx->chip_class >= SI) {
+			/* Write the timestamp from the CP not waiting for
+			 * outstanding draws (top-of-pipe).
+			 */
+			radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
+			radeon_emit(cs, COPY_DATA_COUNT_SEL |
+					COPY_DATA_SRC_SEL(COPY_DATA_TIMESTAMP) |
+					COPY_DATA_DST_SEL(COPY_DATA_MEM_ASYNC));
+			radeon_emit(cs, 0);
+			radeon_emit(cs, 0);
+			radeon_emit(cs, va);
+			radeon_emit(cs, va >> 32);
+		} else {
+			/* Write the timestamp after the last draw is done.
+			 * (bottom-of-pipe)
+			 */
+			r600_gfx_write_event_eop(ctx, EVENT_TYPE_BOTTOM_OF_PIPE_TS,
+						 0, EOP_DATA_SEL_TIMESTAMP,
+						 NULL, va, 0, query->b.type);
+		}
+		break;
+	case PIPE_QUERY_PIPELINE_STATISTICS:
+		radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
+		radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2));
+		radeon_emit(cs, va);
+		radeon_emit(cs, va >> 32);
+		break;
+	default:
+		assert(0);
+	}
+	r600_emit_reloc(ctx, &ctx->gfx, query->buffer.buf, RADEON_USAGE_WRITE,
+			RADEON_PRIO_QUERY);
+}
+
+static void r600_query_hw_emit_start(struct r600_common_context *ctx,
+				     struct r600_query_hw *query)
+{
+	uint64_t va;
+
+	if (!query->buffer.buf)
+		return; // previous buffer allocation failure
+
+	r600_update_occlusion_query_state(ctx, query->b.type, 1);
+	r600_update_prims_generated_query_state(ctx, query->b.type, 1);
+
+	ctx->need_gfx_cs_space(&ctx->b, query->num_cs_dw_begin + query->num_cs_dw_end,
+			       true);
+
+	/* Get a new query buffer if needed. */
+	if (query->buffer.results_end + query->result_size > query->buffer.buf->b.b.width0) {
+		struct r600_query_buffer *qbuf = MALLOC_STRUCT(r600_query_buffer);
+		*qbuf = query->buffer;
+		query->buffer.results_end = 0;
+		query->buffer.previous = qbuf;
+		query->buffer.buf = r600_new_query_buffer(ctx->screen, query);
+		if (!query->buffer.buf)
+			return;
+	}
+
+	/* emit begin query */
+	va = query->buffer.buf->gpu_address + query->buffer.results_end;
+
+	query->ops->emit_start(ctx, query, query->buffer.buf, va);
+
+	ctx->num_cs_dw_queries_suspend += query->num_cs_dw_end;
+}
+
+static void r600_query_hw_do_emit_stop(struct r600_common_context *ctx,
+				       struct r600_query_hw *query,
+				       struct r600_resource *buffer,
+				       uint64_t va)
+{
+	struct radeon_winsys_cs *cs = ctx->gfx.cs;
+	uint64_t fence_va = 0;
+
+	switch (query->b.type) {
+	case PIPE_QUERY_OCCLUSION_COUNTER:
+	case PIPE_QUERY_OCCLUSION_PREDICATE:
+		va += 8;
+		radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
+		radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1));
+		radeon_emit(cs, va);
+		radeon_emit(cs, va >> 32);
+
+		fence_va = va + ctx->screen->info.num_render_backends * 16 - 8;
+		break;
+	case PIPE_QUERY_PRIMITIVES_EMITTED:
+	case PIPE_QUERY_PRIMITIVES_GENERATED:
+	case PIPE_QUERY_SO_STATISTICS:
+	case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
+		va += 16;
+		emit_sample_streamout(cs, va, query->stream);
+		break;
+	case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
+		va += 16;
+		for (unsigned stream = 0; stream < R600_MAX_STREAMS; ++stream)
+			emit_sample_streamout(cs, va + 32 * stream, stream);
+		break;
+	case PIPE_QUERY_TIME_ELAPSED:
+		va += 8;
+		/* fall through */
+	case PIPE_QUERY_TIMESTAMP:
+		r600_gfx_write_event_eop(ctx, EVENT_TYPE_BOTTOM_OF_PIPE_TS,
+					 0, EOP_DATA_SEL_TIMESTAMP, NULL, va,
+					 0, query->b.type);
+		fence_va = va + 8;
+		break;
+	case PIPE_QUERY_PIPELINE_STATISTICS: {
+		unsigned sample_size = (query->result_size - 8) / 2;
+
+		va += sample_size;
+		radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
+		radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2));
+		radeon_emit(cs, va);
+		radeon_emit(cs, va >> 32);
+
+		fence_va = va + sample_size;
+		break;
+	}
+	default:
+		assert(0);
+	}
+	r600_emit_reloc(ctx, &ctx->gfx, query->buffer.buf, RADEON_USAGE_WRITE,
+			RADEON_PRIO_QUERY);
+
+	if (fence_va)
+		r600_gfx_write_event_eop(ctx, EVENT_TYPE_BOTTOM_OF_PIPE_TS, 0,
+					 EOP_DATA_SEL_VALUE_32BIT,
+					 query->buffer.buf, fence_va, 0x80000000,
+					 query->b.type);
+}
+
+static void r600_query_hw_emit_stop(struct r600_common_context *ctx,
+				    struct r600_query_hw *query)
+{
+	uint64_t va;
+
+	if (!query->buffer.buf)
+		return; // previous buffer allocation failure
+
+	/* The queries which need begin already called this in begin_query. */
+	if (query->flags & R600_QUERY_HW_FLAG_NO_START) {
+		ctx->need_gfx_cs_space(&ctx->b, query->num_cs_dw_end, false);
+	}
+
+	/* emit end query */
+	va = query->buffer.buf->gpu_address + query->buffer.results_end;
+
+	query->ops->emit_stop(ctx, query, query->buffer.buf, va);
+
+	query->buffer.results_end += query->result_size;
+
+	if (!(query->flags & R600_QUERY_HW_FLAG_NO_START))
+		ctx->num_cs_dw_queries_suspend -= query->num_cs_dw_end;
+
+	r600_update_occlusion_query_state(ctx, query->b.type, -1);
+	r600_update_prims_generated_query_state(ctx, query->b.type, -1);
+}
+
+static void emit_set_predicate(struct r600_common_context *ctx,
+			       struct r600_resource *buf, uint64_t va,
+			       uint32_t op)
+{
+	struct radeon_winsys_cs *cs = ctx->gfx.cs;
+
+	if (ctx->chip_class >= GFX9) {
+		radeon_emit(cs, PKT3(PKT3_SET_PREDICATION, 2, 0));
+		radeon_emit(cs, op);
+		radeon_emit(cs, va);
+		radeon_emit(cs, va >> 32);
+	} else {
+		radeon_emit(cs, PKT3(PKT3_SET_PREDICATION, 1, 0));
+		radeon_emit(cs, va);
+		radeon_emit(cs, op | ((va >> 32) & 0xFF));
+	}
+	r600_emit_reloc(ctx, &ctx->gfx, buf, RADEON_USAGE_READ,
+			RADEON_PRIO_QUERY);
+}
+
+static void r600_emit_query_predication(struct r600_common_context *ctx,
+					struct r600_atom *atom)
+{
+	struct r600_query_hw *query = (struct r600_query_hw *)ctx->render_cond;
+	struct r600_query_buffer *qbuf;
+	uint32_t op;
+	bool flag_wait, invert;
+
+	if (!query)
+		return;
+
+	invert = ctx->render_cond_invert;
+	flag_wait = ctx->render_cond_mode == PIPE_RENDER_COND_WAIT ||
+		    ctx->render_cond_mode == PIPE_RENDER_COND_BY_REGION_WAIT;
+
+	if (query->workaround_buf) {
+		op = PRED_OP(PREDICATION_OP_BOOL64);
+	} else {
+		switch (query->b.type) {
+		case PIPE_QUERY_OCCLUSION_COUNTER:
+		case PIPE_QUERY_OCCLUSION_PREDICATE:
+			op = PRED_OP(PREDICATION_OP_ZPASS);
+			break;
+		case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
+		case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
+			op = PRED_OP(PREDICATION_OP_PRIMCOUNT);
+			invert = !invert;
+			break;
+		default:
+			assert(0);
+			return;
+		}
+	}
+
+	/* if true then invert, see GL_ARB_conditional_render_inverted */
+	if (invert)
+		op |= PREDICATION_DRAW_NOT_VISIBLE; /* Draw if not visible or overflow */
+	else
+		op |= PREDICATION_DRAW_VISIBLE; /* Draw if visible or no overflow */
+
+	/* Use the value written by compute shader as a workaround. Note that
+	 * the wait flag does not apply in this predication mode.
+	 *
+	 * The shader outputs the result value to L2. Workarounds only affect VI
+	 * and later, where the CP reads data from L2, so we don't need an
+	 * additional flush.
+	 */
+	if (query->workaround_buf) {
+		uint64_t va = query->workaround_buf->gpu_address + query->workaround_offset;
+		emit_set_predicate(ctx, query->workaround_buf, va, op);
+		return;
+	}
+
+	op |= flag_wait ? PREDICATION_HINT_WAIT : PREDICATION_HINT_NOWAIT_DRAW;
+
+	/* emit predicate packets for all data blocks */
+	for (qbuf = &query->buffer; qbuf; qbuf = qbuf->previous) {
+		unsigned results_base = 0;
+		uint64_t va_base = qbuf->buf->gpu_address;
+
+		while (results_base < qbuf->results_end) {
+			uint64_t va = va_base + results_base;
+
+			if (query->b.type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) {
+				for (unsigned stream = 0; stream < R600_MAX_STREAMS; ++stream) {
+					emit_set_predicate(ctx, qbuf->buf, va + 32 * stream, op);
+
+					/* set CONTINUE bit for all packets except the first */
+					op |= PREDICATION_CONTINUE;
+				}
+			} else {
+				emit_set_predicate(ctx, qbuf->buf, va, op);
+				op |= PREDICATION_CONTINUE;
+			}
+
+			results_base += query->result_size;
+		}
+	}
+}
+
+static struct pipe_query *r600_create_query(struct pipe_context *ctx, unsigned query_type, unsigned index)
+{
+	struct r600_common_screen *rscreen =
+		(struct r600_common_screen *)ctx->screen;
+
+	if (query_type == PIPE_QUERY_TIMESTAMP_DISJOINT ||
+	    query_type == PIPE_QUERY_GPU_FINISHED ||
+	    query_type >= PIPE_QUERY_DRIVER_SPECIFIC)
+		return r600_query_sw_create(query_type);
+
+	return r600_query_hw_create(rscreen, query_type, index);
+}
+
+static void r600_destroy_query(struct pipe_context *ctx, struct pipe_query *query)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct r600_query *rquery = (struct r600_query *)query;
+
+	rquery->ops->destroy(rctx->screen, rquery);
+}
+
+static boolean r600_begin_query(struct pipe_context *ctx,
+                                struct pipe_query *query)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct r600_query *rquery = (struct r600_query *)query;
+
+	return rquery->ops->begin(rctx, rquery);
+}
+
+void r600_query_hw_reset_buffers(struct r600_common_context *rctx,
+				 struct r600_query_hw *query)
+{
+	struct r600_query_buffer *prev = query->buffer.previous;
+
+	/* Discard the old query buffers. */
+	while (prev) {
+		struct r600_query_buffer *qbuf = prev;
+		prev = prev->previous;
+		r600_resource_reference(&qbuf->buf, NULL);
+		FREE(qbuf);
+	}
+
+	query->buffer.results_end = 0;
+	query->buffer.previous = NULL;
+
+	/* Obtain a new buffer if the current one can't be mapped without a stall. */
+	if (r600_rings_is_buffer_referenced(rctx, query->buffer.buf->buf, RADEON_USAGE_READWRITE) ||
+	    !rctx->ws->buffer_wait(query->buffer.buf->buf, 0, RADEON_USAGE_READWRITE)) {
+		r600_resource_reference(&query->buffer.buf, NULL);
+		query->buffer.buf = r600_new_query_buffer(rctx->screen, query);
+	} else {
+		if (!query->ops->prepare_buffer(rctx->screen, query, query->buffer.buf))
+			r600_resource_reference(&query->buffer.buf, NULL);
+	}
+}
+
+bool r600_query_hw_begin(struct r600_common_context *rctx,
+			 struct r600_query *rquery)
+{
+	struct r600_query_hw *query = (struct r600_query_hw *)rquery;
+
+	if (query->flags & R600_QUERY_HW_FLAG_NO_START) {
+		assert(0);
+		return false;
+	}
+
+	if (!(query->flags & R600_QUERY_HW_FLAG_BEGIN_RESUMES))
+		r600_query_hw_reset_buffers(rctx, query);
+
+	r600_resource_reference(&query->workaround_buf, NULL);
+
+	r600_query_hw_emit_start(rctx, query);
+	if (!query->buffer.buf)
+		return false;
+
+	LIST_ADDTAIL(&query->list, &rctx->active_queries);
+	return true;
+}
+
+static bool r600_end_query(struct pipe_context *ctx, struct pipe_query *query)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct r600_query *rquery = (struct r600_query *)query;
+
+	return rquery->ops->end(rctx, rquery);
+}
+
+bool r600_query_hw_end(struct r600_common_context *rctx,
+		       struct r600_query *rquery)
+{
+	struct r600_query_hw *query = (struct r600_query_hw *)rquery;
+
+	if (query->flags & R600_QUERY_HW_FLAG_NO_START)
+		r600_query_hw_reset_buffers(rctx, query);
+
+	r600_query_hw_emit_stop(rctx, query);
+
+	if (!(query->flags & R600_QUERY_HW_FLAG_NO_START))
+		LIST_DELINIT(&query->list);
+
+	if (!query->buffer.buf)
+		return false;
+
+	return true;
+}
+
+static void r600_get_hw_query_params(struct r600_common_context *rctx,
+				     struct r600_query_hw *rquery, int index,
+				     struct r600_hw_query_params *params)
+{
+	unsigned max_rbs = rctx->screen->info.num_render_backends;
+
+	params->pair_stride = 0;
+	params->pair_count = 1;
+
+	switch (rquery->b.type) {
+	case PIPE_QUERY_OCCLUSION_COUNTER:
+	case PIPE_QUERY_OCCLUSION_PREDICATE:
+		params->start_offset = 0;
+		params->end_offset = 8;
+		params->fence_offset = max_rbs * 16;
+		params->pair_stride = 16;
+		params->pair_count = max_rbs;
+		break;
+	case PIPE_QUERY_TIME_ELAPSED:
+		params->start_offset = 0;
+		params->end_offset = 8;
+		params->fence_offset = 16;
+		break;
+	case PIPE_QUERY_TIMESTAMP:
+		params->start_offset = 0;
+		params->end_offset = 0;
+		params->fence_offset = 8;
+		break;
+	case PIPE_QUERY_PRIMITIVES_EMITTED:
+		params->start_offset = 8;
+		params->end_offset = 24;
+		params->fence_offset = params->end_offset + 4;
+		break;
+	case PIPE_QUERY_PRIMITIVES_GENERATED:
+		params->start_offset = 0;
+		params->end_offset = 16;
+		params->fence_offset = params->end_offset + 4;
+		break;
+	case PIPE_QUERY_SO_STATISTICS:
+		params->start_offset = 8 - index * 8;
+		params->end_offset = 24 - index * 8;
+		params->fence_offset = params->end_offset + 4;
+		break;
+	case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
+		params->pair_count = R600_MAX_STREAMS;
+		params->pair_stride = 32;
+	case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
+		params->start_offset = 0;
+		params->end_offset = 16;
+
+		/* We can re-use the high dword of the last 64-bit value as a
+		 * fence: it is initialized as 0, and the high bit is set by
+		 * the write of the streamout stats event.
+		 */
+		params->fence_offset = rquery->result_size - 4;
+		break;
+	case PIPE_QUERY_PIPELINE_STATISTICS:
+	{
+		/* Offsets apply to EG+ */
+		static const unsigned offsets[] = {56, 48, 24, 32, 40, 16, 8, 0, 64, 72, 80};
+		params->start_offset = offsets[index];
+		params->end_offset = 88 + offsets[index];
+		params->fence_offset = 2 * 88;
+		break;
+	}
+	default:
+		unreachable("r600_get_hw_query_params unsupported");
+	}
+}
+
+static unsigned r600_query_read_result(void *map, unsigned start_index, unsigned end_index,
+				       bool test_status_bit)
+{
+	uint32_t *current_result = (uint32_t*)map;
+	uint64_t start, end;
+
+	start = (uint64_t)current_result[start_index] |
+		(uint64_t)current_result[start_index+1] << 32;
+	end = (uint64_t)current_result[end_index] |
+	      (uint64_t)current_result[end_index+1] << 32;
+
+	if (!test_status_bit ||
+	    ((start & 0x8000000000000000UL) && (end & 0x8000000000000000UL))) {
+		return end - start;
+	}
+	return 0;
+}
+
+static void r600_query_hw_add_result(struct r600_common_screen *rscreen,
+				     struct r600_query_hw *query,
+				     void *buffer,
+				     union pipe_query_result *result)
+{
+	unsigned max_rbs = rscreen->info.num_render_backends;
+
+	switch (query->b.type) {
+	case PIPE_QUERY_OCCLUSION_COUNTER: {
+		for (unsigned i = 0; i < max_rbs; ++i) {
+			unsigned results_base = i * 16;
+			result->u64 +=
+				r600_query_read_result(buffer + results_base, 0, 2, true);
+		}
+		break;
+	}
+	case PIPE_QUERY_OCCLUSION_PREDICATE: {
+		for (unsigned i = 0; i < max_rbs; ++i) {
+			unsigned results_base = i * 16;
+			result->b = result->b ||
+				r600_query_read_result(buffer + results_base, 0, 2, true) != 0;
+		}
+		break;
+	}
+	case PIPE_QUERY_TIME_ELAPSED:
+		result->u64 += r600_query_read_result(buffer, 0, 2, false);
+		break;
+	case PIPE_QUERY_TIMESTAMP:
+		result->u64 = *(uint64_t*)buffer;
+		break;
+	case PIPE_QUERY_PRIMITIVES_EMITTED:
+		/* SAMPLE_STREAMOUTSTATS stores this structure:
+		 * {
+		 *    u64 NumPrimitivesWritten;
+		 *    u64 PrimitiveStorageNeeded;
+		 * }
+		 * We only need NumPrimitivesWritten here. */
+		result->u64 += r600_query_read_result(buffer, 2, 6, true);
+		break;
+	case PIPE_QUERY_PRIMITIVES_GENERATED:
+		/* Here we read PrimitiveStorageNeeded. */
+		result->u64 += r600_query_read_result(buffer, 0, 4, true);
+		break;
+	case PIPE_QUERY_SO_STATISTICS:
+		result->so_statistics.num_primitives_written +=
+			r600_query_read_result(buffer, 2, 6, true);
+		result->so_statistics.primitives_storage_needed +=
+			r600_query_read_result(buffer, 0, 4, true);
+		break;
+	case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
+		result->b = result->b ||
+			r600_query_read_result(buffer, 2, 6, true) !=
+			r600_query_read_result(buffer, 0, 4, true);
+		break;
+	case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
+		for (unsigned stream = 0; stream < R600_MAX_STREAMS; ++stream) {
+			result->b = result->b ||
+				r600_query_read_result(buffer, 2, 6, true) !=
+				r600_query_read_result(buffer, 0, 4, true);
+			buffer = (char *)buffer + 32;
+		}
+		break;
+	case PIPE_QUERY_PIPELINE_STATISTICS:
+		if (rscreen->chip_class >= EVERGREEN) {
+			result->pipeline_statistics.ps_invocations +=
+				r600_query_read_result(buffer, 0, 22, false);
+			result->pipeline_statistics.c_primitives +=
+				r600_query_read_result(buffer, 2, 24, false);
+			result->pipeline_statistics.c_invocations +=
+				r600_query_read_result(buffer, 4, 26, false);
+			result->pipeline_statistics.vs_invocations +=
+				r600_query_read_result(buffer, 6, 28, false);
+			result->pipeline_statistics.gs_invocations +=
+				r600_query_read_result(buffer, 8, 30, false);
+			result->pipeline_statistics.gs_primitives +=
+				r600_query_read_result(buffer, 10, 32, false);
+			result->pipeline_statistics.ia_primitives +=
+				r600_query_read_result(buffer, 12, 34, false);
+			result->pipeline_statistics.ia_vertices +=
+				r600_query_read_result(buffer, 14, 36, false);
+			result->pipeline_statistics.hs_invocations +=
+				r600_query_read_result(buffer, 16, 38, false);
+			result->pipeline_statistics.ds_invocations +=
+				r600_query_read_result(buffer, 18, 40, false);
+			result->pipeline_statistics.cs_invocations +=
+				r600_query_read_result(buffer, 20, 42, false);
+		} else {
+			result->pipeline_statistics.ps_invocations +=
+				r600_query_read_result(buffer, 0, 16, false);
+			result->pipeline_statistics.c_primitives +=
+				r600_query_read_result(buffer, 2, 18, false);
+			result->pipeline_statistics.c_invocations +=
+				r600_query_read_result(buffer, 4, 20, false);
+			result->pipeline_statistics.vs_invocations +=
+				r600_query_read_result(buffer, 6, 22, false);
+			result->pipeline_statistics.gs_invocations +=
+				r600_query_read_result(buffer, 8, 24, false);
+			result->pipeline_statistics.gs_primitives +=
+				r600_query_read_result(buffer, 10, 26, false);
+			result->pipeline_statistics.ia_primitives +=
+				r600_query_read_result(buffer, 12, 28, false);
+			result->pipeline_statistics.ia_vertices +=
+				r600_query_read_result(buffer, 14, 30, false);
+		}
+#if 0 /* for testing */
+		printf("Pipeline stats: IA verts=%llu, IA prims=%llu, VS=%llu, HS=%llu, "
+		       "DS=%llu, GS=%llu, GS prims=%llu, Clipper=%llu, "
+		       "Clipper prims=%llu, PS=%llu, CS=%llu\n",
+		       result->pipeline_statistics.ia_vertices,
+		       result->pipeline_statistics.ia_primitives,
+		       result->pipeline_statistics.vs_invocations,
+		       result->pipeline_statistics.hs_invocations,
+		       result->pipeline_statistics.ds_invocations,
+		       result->pipeline_statistics.gs_invocations,
+		       result->pipeline_statistics.gs_primitives,
+		       result->pipeline_statistics.c_invocations,
+		       result->pipeline_statistics.c_primitives,
+		       result->pipeline_statistics.ps_invocations,
+		       result->pipeline_statistics.cs_invocations);
+#endif
+		break;
+	default:
+		assert(0);
+	}
+}
+
+static boolean r600_get_query_result(struct pipe_context *ctx,
+				     struct pipe_query *query, boolean wait,
+				     union pipe_query_result *result)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct r600_query *rquery = (struct r600_query *)query;
+
+	return rquery->ops->get_result(rctx, rquery, wait, result);
+}
+
+static void r600_get_query_result_resource(struct pipe_context *ctx,
+                                           struct pipe_query *query,
+                                           boolean wait,
+                                           enum pipe_query_value_type result_type,
+                                           int index,
+                                           struct pipe_resource *resource,
+                                           unsigned offset)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct r600_query *rquery = (struct r600_query *)query;
+
+	rquery->ops->get_result_resource(rctx, rquery, wait, result_type, index,
+	                                 resource, offset);
+}
+
+static void r600_query_hw_clear_result(struct r600_query_hw *query,
+				       union pipe_query_result *result)
+{
+	util_query_clear_result(result, query->b.type);
+}
+
+bool r600_query_hw_get_result(struct r600_common_context *rctx,
+			      struct r600_query *rquery,
+			      bool wait, union pipe_query_result *result)
+{
+	struct r600_common_screen *rscreen = rctx->screen;
+	struct r600_query_hw *query = (struct r600_query_hw *)rquery;
+	struct r600_query_buffer *qbuf;
+
+	query->ops->clear_result(query, result);
+
+	for (qbuf = &query->buffer; qbuf; qbuf = qbuf->previous) {
+		unsigned usage = PIPE_TRANSFER_READ |
+				 (wait ? 0 : PIPE_TRANSFER_DONTBLOCK);
+		unsigned results_base = 0;
+		void *map;
+
+		if (rquery->b.flushed)
+			map = rctx->ws->buffer_map(qbuf->buf->buf, NULL, usage);
+		else
+			map = r600_buffer_map_sync_with_rings(rctx, qbuf->buf, usage);
+
+		if (!map)
+			return false;
+
+		while (results_base != qbuf->results_end) {
+			query->ops->add_result(rscreen, query, map + results_base,
+					       result);
+			results_base += query->result_size;
+		}
+	}
+
+	/* Convert the time to expected units. */
+	if (rquery->type == PIPE_QUERY_TIME_ELAPSED ||
+	    rquery->type == PIPE_QUERY_TIMESTAMP) {
+		result->u64 = (1000000 * result->u64) / rscreen->info.clock_crystal_freq;
+	}
+	return true;
+}
+
+/* Create the compute shader that is used to collect the results.
+ *
+ * One compute grid with a single thread is launched for every query result
+ * buffer. The thread (optionally) reads a previous summary buffer, then
+ * accumulates data from the query result buffer, and writes the result either
+ * to a summary buffer to be consumed by the next grid invocation or to the
+ * user-supplied buffer.
+ *
+ * Data layout:
+ *
+ * CONST
+ *  0.x = end_offset
+ *  0.y = result_stride
+ *  0.z = result_count
+ *  0.w = bit field:
+ *          1: read previously accumulated values
+ *          2: write accumulated values for chaining
+ *          4: write result available
+ *          8: convert result to boolean (0/1)
+ *         16: only read one dword and use that as result
+ *         32: apply timestamp conversion
+ *         64: store full 64 bits result
+ *        128: store signed 32 bits result
+ *        256: SO_OVERFLOW mode: take the difference of two successive half-pairs
+ *  1.x = fence_offset
+ *  1.y = pair_stride
+ *  1.z = pair_count
+ *
+ * BUFFER[0] = query result buffer
+ * BUFFER[1] = previous summary buffer
+ * BUFFER[2] = next summary buffer or user-supplied buffer
+ */
+static void r600_create_query_result_shader(struct r600_common_context *rctx)
+{
+	/* TEMP[0].xy = accumulated result so far
+	 * TEMP[0].z = result not available
+	 *
+	 * TEMP[1].x = current result index
+	 * TEMP[1].y = current pair index
+	 */
+	static const char text_tmpl[] =
+		"COMP\n"
+		"PROPERTY CS_FIXED_BLOCK_WIDTH 1\n"
+		"PROPERTY CS_FIXED_BLOCK_HEIGHT 1\n"
+		"PROPERTY CS_FIXED_BLOCK_DEPTH 1\n"
+		"DCL BUFFER[0]\n"
+		"DCL BUFFER[1]\n"
+		"DCL BUFFER[2]\n"
+		"DCL CONST[0][0..1]\n"
+		"DCL TEMP[0..5]\n"
+		"IMM[0] UINT32 {0, 31, 2147483647, 4294967295}\n"
+		"IMM[1] UINT32 {1, 2, 4, 8}\n"
+		"IMM[2] UINT32 {16, 32, 64, 128}\n"
+		"IMM[3] UINT32 {1000000, 0, %u, 0}\n" /* for timestamp conversion */
+		"IMM[4] UINT32 {256, 0, 0, 0}\n"
+
+		"AND TEMP[5], CONST[0][0].wwww, IMM[2].xxxx\n"
+		"UIF TEMP[5]\n"
+			/* Check result availability. */
+			"LOAD TEMP[1].x, BUFFER[0], CONST[0][1].xxxx\n"
+			"ISHR TEMP[0].z, TEMP[1].xxxx, IMM[0].yyyy\n"
+			"MOV TEMP[1], TEMP[0].zzzz\n"
+			"NOT TEMP[0].z, TEMP[0].zzzz\n"
+
+			/* Load result if available. */
+			"UIF TEMP[1]\n"
+				"LOAD TEMP[0].xy, BUFFER[0], IMM[0].xxxx\n"
+			"ENDIF\n"
+		"ELSE\n"
+			/* Load previously accumulated result if requested. */
+			"MOV TEMP[0], IMM[0].xxxx\n"
+			"AND TEMP[4], CONST[0][0].wwww, IMM[1].xxxx\n"
+			"UIF TEMP[4]\n"
+				"LOAD TEMP[0].xyz, BUFFER[1], IMM[0].xxxx\n"
+			"ENDIF\n"
+
+			"MOV TEMP[1].x, IMM[0].xxxx\n"
+			"BGNLOOP\n"
+				/* Break if accumulated result so far is not available. */
+				"UIF TEMP[0].zzzz\n"
+					"BRK\n"
+				"ENDIF\n"
+
+				/* Break if result_index >= result_count. */
+				"USGE TEMP[5], TEMP[1].xxxx, CONST[0][0].zzzz\n"
+				"UIF TEMP[5]\n"
+					"BRK\n"
+				"ENDIF\n"
+
+				/* Load fence and check result availability */
+				"UMAD TEMP[5].x, TEMP[1].xxxx, CONST[0][0].yyyy, CONST[0][1].xxxx\n"
+				"LOAD TEMP[5].x, BUFFER[0], TEMP[5].xxxx\n"
+				"ISHR TEMP[0].z, TEMP[5].xxxx, IMM[0].yyyy\n"
+				"NOT TEMP[0].z, TEMP[0].zzzz\n"
+				"UIF TEMP[0].zzzz\n"
+					"BRK\n"
+				"ENDIF\n"
+
+				"MOV TEMP[1].y, IMM[0].xxxx\n"
+				"BGNLOOP\n"
+					/* Load start and end. */
+					"UMUL TEMP[5].x, TEMP[1].xxxx, CONST[0][0].yyyy\n"
+					"UMAD TEMP[5].x, TEMP[1].yyyy, CONST[0][1].yyyy, TEMP[5].xxxx\n"
+					"LOAD TEMP[2].xy, BUFFER[0], TEMP[5].xxxx\n"
+
+					"UADD TEMP[5].y, TEMP[5].xxxx, CONST[0][0].xxxx\n"
+					"LOAD TEMP[3].xy, BUFFER[0], TEMP[5].yyyy\n"
+
+					"U64ADD TEMP[4].xy, TEMP[3], -TEMP[2]\n"
+
+					"AND TEMP[5].z, CONST[0][0].wwww, IMM[4].xxxx\n"
+					"UIF TEMP[5].zzzz\n"
+						/* Load second start/end half-pair and
+						 * take the difference
+						 */
+						"UADD TEMP[5].xy, TEMP[5], IMM[1].wwww\n"
+						"LOAD TEMP[2].xy, BUFFER[0], TEMP[5].xxxx\n"
+						"LOAD TEMP[3].xy, BUFFER[0], TEMP[5].yyyy\n"
+
+						"U64ADD TEMP[3].xy, TEMP[3], -TEMP[2]\n"
+						"U64ADD TEMP[4].xy, TEMP[4], -TEMP[3]\n"
+					"ENDIF\n"
+
+					"U64ADD TEMP[0].xy, TEMP[0], TEMP[4]\n"
+
+					/* Increment pair index */
+					"UADD TEMP[1].y, TEMP[1].yyyy, IMM[1].xxxx\n"
+					"USGE TEMP[5], TEMP[1].yyyy, CONST[0][1].zzzz\n"
+					"UIF TEMP[5]\n"
+						"BRK\n"
+					"ENDIF\n"
+				"ENDLOOP\n"
+
+				/* Increment result index */
+				"UADD TEMP[1].x, TEMP[1].xxxx, IMM[1].xxxx\n"
+			"ENDLOOP\n"
+		"ENDIF\n"
+
+		"AND TEMP[4], CONST[0][0].wwww, IMM[1].yyyy\n"
+		"UIF TEMP[4]\n"
+			/* Store accumulated data for chaining. */
+			"STORE BUFFER[2].xyz, IMM[0].xxxx, TEMP[0]\n"
+		"ELSE\n"
+			"AND TEMP[4], CONST[0][0].wwww, IMM[1].zzzz\n"
+			"UIF TEMP[4]\n"
+				/* Store result availability. */
+				"NOT TEMP[0].z, TEMP[0]\n"
+				"AND TEMP[0].z, TEMP[0].zzzz, IMM[1].xxxx\n"
+				"STORE BUFFER[2].x, IMM[0].xxxx, TEMP[0].zzzz\n"
+
+				"AND TEMP[4], CONST[0][0].wwww, IMM[2].zzzz\n"
+				"UIF TEMP[4]\n"
+					"STORE BUFFER[2].y, IMM[0].xxxx, IMM[0].xxxx\n"
+				"ENDIF\n"
+			"ELSE\n"
+				/* Store result if it is available. */
+				"NOT TEMP[4], TEMP[0].zzzz\n"
+				"UIF TEMP[4]\n"
+					/* Apply timestamp conversion */
+					"AND TEMP[4], CONST[0][0].wwww, IMM[2].yyyy\n"
+					"UIF TEMP[4]\n"
+						"U64MUL TEMP[0].xy, TEMP[0], IMM[3].xyxy\n"
+						"U64DIV TEMP[0].xy, TEMP[0], IMM[3].zwzw\n"
+					"ENDIF\n"
+
+					/* Convert to boolean */
+					"AND TEMP[4], CONST[0][0].wwww, IMM[1].wwww\n"
+					"UIF TEMP[4]\n"
+						"U64SNE TEMP[0].x, TEMP[0].xyxy, IMM[4].zwzw\n"
+						"AND TEMP[0].x, TEMP[0].xxxx, IMM[1].xxxx\n"
+						"MOV TEMP[0].y, IMM[0].xxxx\n"
+					"ENDIF\n"
+
+					"AND TEMP[4], CONST[0][0].wwww, IMM[2].zzzz\n"
+					"UIF TEMP[4]\n"
+						"STORE BUFFER[2].xy, IMM[0].xxxx, TEMP[0].xyxy\n"
+					"ELSE\n"
+						/* Clamping */
+						"UIF TEMP[0].yyyy\n"
+							"MOV TEMP[0].x, IMM[0].wwww\n"
+						"ENDIF\n"
+
+						"AND TEMP[4], CONST[0][0].wwww, IMM[2].wwww\n"
+						"UIF TEMP[4]\n"
+							"UMIN TEMP[0].x, TEMP[0].xxxx, IMM[0].zzzz\n"
+						"ENDIF\n"
+
+						"STORE BUFFER[2].x, IMM[0].xxxx, TEMP[0].xxxx\n"
+					"ENDIF\n"
+				"ENDIF\n"
+			"ENDIF\n"
+		"ENDIF\n"
+
+		"END\n";
+
+	char text[sizeof(text_tmpl) + 32];
+	struct tgsi_token tokens[1024];
+	struct pipe_compute_state state = {};
+
+	/* Hard code the frequency into the shader so that the backend can
+	 * use the full range of optimizations for divide-by-constant.
+	 */
+	snprintf(text, sizeof(text), text_tmpl,
+		 rctx->screen->info.clock_crystal_freq);
+
+	if (!tgsi_text_translate(text, tokens, ARRAY_SIZE(tokens))) {
+		assert(false);
+		return;
+	}
+
+	state.ir_type = PIPE_SHADER_IR_TGSI;
+	state.prog = tokens;
+
+	rctx->query_result_shader = rctx->b.create_compute_state(&rctx->b, &state);
+}
+
+static void r600_restore_qbo_state(struct r600_common_context *rctx,
+				   struct r600_qbo_state *st)
+{
+	rctx->b.bind_compute_state(&rctx->b, st->saved_compute);
+
+	rctx->b.set_constant_buffer(&rctx->b, PIPE_SHADER_COMPUTE, 0, &st->saved_const0);
+	pipe_resource_reference(&st->saved_const0.buffer, NULL);
+
+	rctx->b.set_shader_buffers(&rctx->b, PIPE_SHADER_COMPUTE, 0, 3, st->saved_ssbo);
+	for (unsigned i = 0; i < 3; ++i)
+		pipe_resource_reference(&st->saved_ssbo[i].buffer, NULL);
+}
+
+static void r600_query_hw_get_result_resource(struct r600_common_context *rctx,
+                                              struct r600_query *rquery,
+                                              bool wait,
+                                              enum pipe_query_value_type result_type,
+                                              int index,
+                                              struct pipe_resource *resource,
+                                              unsigned offset)
+{
+	struct r600_query_hw *query = (struct r600_query_hw *)rquery;
+	struct r600_query_buffer *qbuf;
+	struct r600_query_buffer *qbuf_prev;
+	struct pipe_resource *tmp_buffer = NULL;
+	unsigned tmp_buffer_offset = 0;
+	struct r600_qbo_state saved_state = {};
+	struct pipe_grid_info grid = {};
+	struct pipe_constant_buffer constant_buffer = {};
+	struct pipe_shader_buffer ssbo[3];
+	struct r600_hw_query_params params;
+	struct {
+		uint32_t end_offset;
+		uint32_t result_stride;
+		uint32_t result_count;
+		uint32_t config;
+		uint32_t fence_offset;
+		uint32_t pair_stride;
+		uint32_t pair_count;
+	} consts;
+
+	if (!rctx->query_result_shader) {
+		r600_create_query_result_shader(rctx);
+		if (!rctx->query_result_shader)
+			return;
+	}
+
+	if (query->buffer.previous) {
+		u_suballocator_alloc(rctx->allocator_zeroed_memory, 16, 16,
+				     &tmp_buffer_offset, &tmp_buffer);
+		if (!tmp_buffer)
+			return;
+	}
+
+	rctx->save_qbo_state(&rctx->b, &saved_state);
+
+	r600_get_hw_query_params(rctx, query, index >= 0 ? index : 0, &params);
+	consts.end_offset = params.end_offset - params.start_offset;
+	consts.fence_offset = params.fence_offset - params.start_offset;
+	consts.result_stride = query->result_size;
+	consts.pair_stride = params.pair_stride;
+	consts.pair_count = params.pair_count;
+
+	constant_buffer.buffer_size = sizeof(consts);
+	constant_buffer.user_buffer = &consts;
+
+	ssbo[1].buffer = tmp_buffer;
+	ssbo[1].buffer_offset = tmp_buffer_offset;
+	ssbo[1].buffer_size = 16;
+
+	ssbo[2] = ssbo[1];
+
+	rctx->b.bind_compute_state(&rctx->b, rctx->query_result_shader);
+
+	grid.block[0] = 1;
+	grid.block[1] = 1;
+	grid.block[2] = 1;
+	grid.grid[0] = 1;
+	grid.grid[1] = 1;
+	grid.grid[2] = 1;
+
+	consts.config = 0;
+	if (index < 0)
+		consts.config |= 4;
+	if (query->b.type == PIPE_QUERY_OCCLUSION_PREDICATE)
+		consts.config |= 8;
+	else if (query->b.type == PIPE_QUERY_SO_OVERFLOW_PREDICATE ||
+		 query->b.type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE)
+		consts.config |= 8 | 256;
+	else if (query->b.type == PIPE_QUERY_TIMESTAMP ||
+		 query->b.type == PIPE_QUERY_TIME_ELAPSED)
+		consts.config |= 32;
+
+	switch (result_type) {
+	case PIPE_QUERY_TYPE_U64:
+	case PIPE_QUERY_TYPE_I64:
+		consts.config |= 64;
+		break;
+	case PIPE_QUERY_TYPE_I32:
+		consts.config |= 128;
+		break;
+	case PIPE_QUERY_TYPE_U32:
+		break;
+	}
+
+	rctx->flags |= rctx->screen->barrier_flags.cp_to_L2;
+
+	for (qbuf = &query->buffer; qbuf; qbuf = qbuf_prev) {
+		if (query->b.type != PIPE_QUERY_TIMESTAMP) {
+			qbuf_prev = qbuf->previous;
+			consts.result_count = qbuf->results_end / query->result_size;
+			consts.config &= ~3;
+			if (qbuf != &query->buffer)
+				consts.config |= 1;
+			if (qbuf->previous)
+				consts.config |= 2;
+		} else {
+			/* Only read the last timestamp. */
+			qbuf_prev = NULL;
+			consts.result_count = 0;
+			consts.config |= 16;
+			params.start_offset += qbuf->results_end - query->result_size;
+		}
+
+		rctx->b.set_constant_buffer(&rctx->b, PIPE_SHADER_COMPUTE, 0, &constant_buffer);
+
+		ssbo[0].buffer = &qbuf->buf->b.b;
+		ssbo[0].buffer_offset = params.start_offset;
+		ssbo[0].buffer_size = qbuf->results_end - params.start_offset;
+
+		if (!qbuf->previous) {
+			ssbo[2].buffer = resource;
+			ssbo[2].buffer_offset = offset;
+			ssbo[2].buffer_size = 8;
+
+			((struct r600_resource *)resource)->TC_L2_dirty = true;
+		}
+
+		rctx->b.set_shader_buffers(&rctx->b, PIPE_SHADER_COMPUTE, 0, 3, ssbo);
+
+		if (wait && qbuf == &query->buffer) {
+			uint64_t va;
+
+			/* Wait for result availability. Wait only for readiness
+			 * of the last entry, since the fence writes should be
+			 * serialized in the CP.
+			 */
+			va = qbuf->buf->gpu_address + qbuf->results_end - query->result_size;
+			va += params.fence_offset;
+
+			r600_gfx_wait_fence(rctx, va, 0x80000000, 0x80000000);
+		}
+
+		rctx->b.launch_grid(&rctx->b, &grid);
+		rctx->flags |= rctx->screen->barrier_flags.compute_to_L2;
+	}
+
+	r600_restore_qbo_state(rctx, &saved_state);
+	pipe_resource_reference(&tmp_buffer, NULL);
+}
+
+static void r600_render_condition(struct pipe_context *ctx,
+				  struct pipe_query *query,
+				  boolean condition,
+				  enum pipe_render_cond_flag mode)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct r600_query_hw *rquery = (struct r600_query_hw *)query;
+	struct r600_query_buffer *qbuf;
+	struct r600_atom *atom = &rctx->render_cond_atom;
+
+	/* Compute the size of SET_PREDICATION packets. */
+	atom->num_dw = 0;
+	if (query) {
+		bool needs_workaround = false;
+
+		/* There is a firmware regression in VI which causes successive
+		 * SET_PREDICATION packets to give the wrong answer for
+		 * non-inverted stream overflow predication.
+		 */
+		if (rctx->chip_class >= VI && !condition &&
+		    (rquery->b.type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE ||
+		     (rquery->b.type == PIPE_QUERY_SO_OVERFLOW_PREDICATE &&
+		      (rquery->buffer.previous ||
+		       rquery->buffer.results_end > rquery->result_size)))) {
+			needs_workaround = true;
+		}
+
+		if (needs_workaround && !rquery->workaround_buf) {
+			bool old_force_off = rctx->render_cond_force_off;
+			rctx->render_cond_force_off = true;
+
+			u_suballocator_alloc(
+				rctx->allocator_zeroed_memory, 8, 8,
+				&rquery->workaround_offset,
+				(struct pipe_resource **)&rquery->workaround_buf);
+
+			/* Reset to NULL to avoid a redundant SET_PREDICATION
+			 * from launching the compute grid.
+			 */
+			rctx->render_cond = NULL;
+
+			ctx->get_query_result_resource(
+				ctx, query, true, PIPE_QUERY_TYPE_U64, 0,
+				&rquery->workaround_buf->b.b, rquery->workaround_offset);
+
+			/* Settings this in the render cond atom is too late,
+			 * so set it here. */
+			rctx->flags |= rctx->screen->barrier_flags.L2_to_cp |
+				       R600_CONTEXT_FLUSH_FOR_RENDER_COND;
+
+			rctx->render_cond_force_off = old_force_off;
+		}
+
+		if (needs_workaround) {
+			atom->num_dw = 5;
+		} else {
+			for (qbuf = &rquery->buffer; qbuf; qbuf = qbuf->previous)
+				atom->num_dw += (qbuf->results_end / rquery->result_size) * 5;
+
+			if (rquery->b.type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE)
+				atom->num_dw *= R600_MAX_STREAMS;
+		}
+	}
+
+	rctx->render_cond = query;
+	rctx->render_cond_invert = condition;
+	rctx->render_cond_mode = mode;
+
+	rctx->set_atom_dirty(rctx, atom, query != NULL);
+}
+
+void r600_suspend_queries(struct r600_common_context *ctx)
+{
+	struct r600_query_hw *query;
+
+	LIST_FOR_EACH_ENTRY(query, &ctx->active_queries, list) {
+		r600_query_hw_emit_stop(ctx, query);
+	}
+	assert(ctx->num_cs_dw_queries_suspend == 0);
+}
+
+static unsigned r600_queries_num_cs_dw_for_resuming(struct r600_common_context *ctx,
+						    struct list_head *query_list)
+{
+	struct r600_query_hw *query;
+	unsigned num_dw = 0;
+
+	LIST_FOR_EACH_ENTRY(query, query_list, list) {
+		/* begin + end */
+		num_dw += query->num_cs_dw_begin + query->num_cs_dw_end;
+
+		/* Workaround for the fact that
+		 * num_cs_dw_nontimer_queries_suspend is incremented for every
+		 * resumed query, which raises the bar in need_cs_space for
+		 * queries about to be resumed.
+		 */
+		num_dw += query->num_cs_dw_end;
+	}
+	/* primitives generated query */
+	num_dw += ctx->streamout.enable_atom.num_dw;
+	/* guess for ZPASS enable or PERFECT_ZPASS_COUNT enable updates */
+	num_dw += 13;
+
+	return num_dw;
+}
+
+void r600_resume_queries(struct r600_common_context *ctx)
+{
+	struct r600_query_hw *query;
+	unsigned num_cs_dw = r600_queries_num_cs_dw_for_resuming(ctx, &ctx->active_queries);
+
+	assert(ctx->num_cs_dw_queries_suspend == 0);
+
+	/* Check CS space here. Resuming must not be interrupted by flushes. */
+	ctx->need_gfx_cs_space(&ctx->b, num_cs_dw, true);
+
+	LIST_FOR_EACH_ENTRY(query, &ctx->active_queries, list) {
+		r600_query_hw_emit_start(ctx, query);
+	}
+}
+
+/* Fix radeon_info::enabled_rb_mask for R600, R700, EVERGREEN, NI. */
+void r600_query_fix_enabled_rb_mask(struct r600_common_screen *rscreen)
+{
+	struct r600_common_context *ctx =
+		(struct r600_common_context*)rscreen->aux_context;
+	struct radeon_winsys_cs *cs = ctx->gfx.cs;
+	struct r600_resource *buffer;
+	uint32_t *results;
+	unsigned i, mask = 0;
+	unsigned max_rbs = ctx->screen->info.num_render_backends;
+
+	assert(rscreen->chip_class <= CAYMAN);
+
+	/* if backend_map query is supported by the kernel */
+	if (rscreen->info.r600_gb_backend_map_valid) {
+		unsigned num_tile_pipes = rscreen->info.num_tile_pipes;
+		unsigned backend_map = rscreen->info.r600_gb_backend_map;
+		unsigned item_width, item_mask;
+
+		if (ctx->chip_class >= EVERGREEN) {
+			item_width = 4;
+			item_mask = 0x7;
+		} else {
+			item_width = 2;
+			item_mask = 0x3;
+		}
+
+		while (num_tile_pipes--) {
+			i = backend_map & item_mask;
+			mask |= (1<<i);
+			backend_map >>= item_width;
+		}
+		if (mask != 0) {
+			rscreen->info.enabled_rb_mask = mask;
+			return;
+		}
+	}
+
+	/* otherwise backup path for older kernels */
+
+	/* create buffer for event data */
+	buffer = (struct r600_resource*)
+		pipe_buffer_create(ctx->b.screen, 0,
+				   PIPE_USAGE_STAGING, max_rbs * 16);
+	if (!buffer)
+		return;
+
+	/* initialize buffer with zeroes */
+	results = r600_buffer_map_sync_with_rings(ctx, buffer, PIPE_TRANSFER_WRITE);
+	if (results) {
+		memset(results, 0, max_rbs * 4 * 4);
+
+		/* emit EVENT_WRITE for ZPASS_DONE */
+		radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
+		radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1));
+		radeon_emit(cs, buffer->gpu_address);
+		radeon_emit(cs, buffer->gpu_address >> 32);
+
+		r600_emit_reloc(ctx, &ctx->gfx, buffer,
+                                RADEON_USAGE_WRITE, RADEON_PRIO_QUERY);
+
+		/* analyze results */
+		results = r600_buffer_map_sync_with_rings(ctx, buffer, PIPE_TRANSFER_READ);
+		if (results) {
+			for(i = 0; i < max_rbs; i++) {
+				/* at least highest bit will be set if backend is used */
+				if (results[i*4 + 1])
+					mask |= (1<<i);
+			}
+		}
+	}
+
+	r600_resource_reference(&buffer, NULL);
+
+	if (mask)
+		rscreen->info.enabled_rb_mask = mask;
+}
+
+#define XFULL(name_, query_type_, type_, result_type_, group_id_) \
+	{ \
+		.name = name_, \
+		.query_type = R600_QUERY_##query_type_, \
+		.type = PIPE_DRIVER_QUERY_TYPE_##type_, \
+		.result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_##result_type_, \
+		.group_id = group_id_ \
+	}
+
+#define X(name_, query_type_, type_, result_type_) \
+	XFULL(name_, query_type_, type_, result_type_, ~(unsigned)0)
+
+#define XG(group_, name_, query_type_, type_, result_type_) \
+	XFULL(name_, query_type_, type_, result_type_, R600_QUERY_GROUP_##group_)
+
+static struct pipe_driver_query_info r600_driver_query_list[] = {
+	X("num-compilations",		NUM_COMPILATIONS,	UINT64, CUMULATIVE),
+	X("num-shaders-created",	NUM_SHADERS_CREATED,	UINT64, CUMULATIVE),
+	X("num-shader-cache-hits",	NUM_SHADER_CACHE_HITS,	UINT64, CUMULATIVE),
+	X("draw-calls",			DRAW_CALLS,		UINT64, AVERAGE),
+	X("decompress-calls",		DECOMPRESS_CALLS,	UINT64, AVERAGE),
+	X("MRT-draw-calls",		MRT_DRAW_CALLS,		UINT64, AVERAGE),
+	X("prim-restart-calls",		PRIM_RESTART_CALLS,	UINT64, AVERAGE),
+	X("spill-draw-calls",		SPILL_DRAW_CALLS,	UINT64, AVERAGE),
+	X("compute-calls",		COMPUTE_CALLS,		UINT64, AVERAGE),
+	X("spill-compute-calls",	SPILL_COMPUTE_CALLS,	UINT64, AVERAGE),
+	X("dma-calls",			DMA_CALLS,		UINT64, AVERAGE),
+	X("cp-dma-calls",		CP_DMA_CALLS,		UINT64, AVERAGE),
+	X("num-vs-flushes",		NUM_VS_FLUSHES,		UINT64, AVERAGE),
+	X("num-ps-flushes",		NUM_PS_FLUSHES,		UINT64, AVERAGE),
+	X("num-cs-flushes",		NUM_CS_FLUSHES,		UINT64, AVERAGE),
+	X("num-CB-cache-flushes",	NUM_CB_CACHE_FLUSHES,	UINT64, AVERAGE),
+	X("num-DB-cache-flushes",	NUM_DB_CACHE_FLUSHES,	UINT64, AVERAGE),
+	X("num-L2-invalidates",		NUM_L2_INVALIDATES,	UINT64, AVERAGE),
+	X("num-L2-writebacks",		NUM_L2_WRITEBACKS,	UINT64, AVERAGE),
+	X("num-resident-handles",	NUM_RESIDENT_HANDLES,	UINT64, AVERAGE),
+	X("tc-offloaded-slots",		TC_OFFLOADED_SLOTS,     UINT64, AVERAGE),
+	X("tc-direct-slots",		TC_DIRECT_SLOTS,	UINT64, AVERAGE),
+	X("tc-num-syncs",		TC_NUM_SYNCS,		UINT64, AVERAGE),
+	X("CS-thread-busy",		CS_THREAD_BUSY,		UINT64, AVERAGE),
+	X("gallium-thread-busy",	GALLIUM_THREAD_BUSY,	UINT64, AVERAGE),
+	X("requested-VRAM",		REQUESTED_VRAM,		BYTES, AVERAGE),
+	X("requested-GTT",		REQUESTED_GTT,		BYTES, AVERAGE),
+	X("mapped-VRAM",		MAPPED_VRAM,		BYTES, AVERAGE),
+	X("mapped-GTT",			MAPPED_GTT,		BYTES, AVERAGE),
+	X("buffer-wait-time",		BUFFER_WAIT_TIME,	MICROSECONDS, CUMULATIVE),
+	X("num-mapped-buffers",		NUM_MAPPED_BUFFERS,	UINT64, AVERAGE),
+	X("num-GFX-IBs",		NUM_GFX_IBS,		UINT64, AVERAGE),
+	X("num-SDMA-IBs",		NUM_SDMA_IBS,		UINT64, AVERAGE),
+	X("GFX-BO-list-size",		GFX_BO_LIST_SIZE,	UINT64, AVERAGE),
+	X("num-bytes-moved",		NUM_BYTES_MOVED,	BYTES, CUMULATIVE),
+	X("num-evictions",		NUM_EVICTIONS,		UINT64, CUMULATIVE),
+	X("VRAM-CPU-page-faults",	NUM_VRAM_CPU_PAGE_FAULTS, UINT64, CUMULATIVE),
+	X("VRAM-usage",			VRAM_USAGE,		BYTES, AVERAGE),
+	X("VRAM-vis-usage",		VRAM_VIS_USAGE,		BYTES, AVERAGE),
+	X("GTT-usage",			GTT_USAGE,		BYTES, AVERAGE),
+	X("back-buffer-ps-draw-ratio",	BACK_BUFFER_PS_DRAW_RATIO, UINT64, AVERAGE),
+
+	/* GPIN queries are for the benefit of old versions of GPUPerfStudio,
+	 * which use it as a fallback path to detect the GPU type.
+	 *
+	 * Note: The names of these queries are significant for GPUPerfStudio
+	 * (and possibly their order as well). */
+	XG(GPIN, "GPIN_000",		GPIN_ASIC_ID,		UINT, AVERAGE),
+	XG(GPIN, "GPIN_001",		GPIN_NUM_SIMD,		UINT, AVERAGE),
+	XG(GPIN, "GPIN_002",		GPIN_NUM_RB,		UINT, AVERAGE),
+	XG(GPIN, "GPIN_003",		GPIN_NUM_SPI,		UINT, AVERAGE),
+	XG(GPIN, "GPIN_004",		GPIN_NUM_SE,		UINT, AVERAGE),
+
+	X("temperature",		GPU_TEMPERATURE,	UINT64, AVERAGE),
+	X("shader-clock",		CURRENT_GPU_SCLK,	HZ, AVERAGE),
+	X("memory-clock",		CURRENT_GPU_MCLK,	HZ, AVERAGE),
+
+	/* The following queries must be at the end of the list because their
+	 * availability is adjusted dynamically based on the DRM version. */
+	X("GPU-load",			GPU_LOAD,		UINT64, AVERAGE),
+	X("GPU-shaders-busy",		GPU_SHADERS_BUSY,	UINT64, AVERAGE),
+	X("GPU-ta-busy",		GPU_TA_BUSY,		UINT64, AVERAGE),
+	X("GPU-gds-busy",		GPU_GDS_BUSY,		UINT64, AVERAGE),
+	X("GPU-vgt-busy",		GPU_VGT_BUSY,		UINT64, AVERAGE),
+	X("GPU-ia-busy",		GPU_IA_BUSY,		UINT64, AVERAGE),
+	X("GPU-sx-busy",		GPU_SX_BUSY,		UINT64, AVERAGE),
+	X("GPU-wd-busy",		GPU_WD_BUSY,		UINT64, AVERAGE),
+	X("GPU-bci-busy",		GPU_BCI_BUSY,		UINT64, AVERAGE),
+	X("GPU-sc-busy",		GPU_SC_BUSY,		UINT64, AVERAGE),
+	X("GPU-pa-busy",		GPU_PA_BUSY,		UINT64, AVERAGE),
+	X("GPU-db-busy",		GPU_DB_BUSY,		UINT64, AVERAGE),
+	X("GPU-cp-busy",		GPU_CP_BUSY,		UINT64, AVERAGE),
+	X("GPU-cb-busy",		GPU_CB_BUSY,		UINT64, AVERAGE),
+	X("GPU-sdma-busy",		GPU_SDMA_BUSY,		UINT64, AVERAGE),
+	X("GPU-pfp-busy",		GPU_PFP_BUSY,		UINT64, AVERAGE),
+	X("GPU-meq-busy",		GPU_MEQ_BUSY,		UINT64, AVERAGE),
+	X("GPU-me-busy",		GPU_ME_BUSY,		UINT64, AVERAGE),
+	X("GPU-surf-sync-busy",		GPU_SURF_SYNC_BUSY,	UINT64, AVERAGE),
+	X("GPU-cp-dma-busy",		GPU_CP_DMA_BUSY,	UINT64, AVERAGE),
+	X("GPU-scratch-ram-busy",	GPU_SCRATCH_RAM_BUSY,	UINT64, AVERAGE),
+};
+
+#undef X
+#undef XG
+#undef XFULL
+
+static unsigned r600_get_num_queries(struct r600_common_screen *rscreen)
+{
+	if (rscreen->info.drm_major == 2 && rscreen->info.drm_minor >= 42)
+		return ARRAY_SIZE(r600_driver_query_list);
+	else if (rscreen->info.drm_major == 3) {
+		if (rscreen->chip_class >= VI)
+			return ARRAY_SIZE(r600_driver_query_list);
+		else
+			return ARRAY_SIZE(r600_driver_query_list) - 7;
+	}
+	else
+		return ARRAY_SIZE(r600_driver_query_list) - 25;
+}
+
+static int r600_get_driver_query_info(struct pipe_screen *screen,
+				      unsigned index,
+				      struct pipe_driver_query_info *info)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+	unsigned num_queries = r600_get_num_queries(rscreen);
+
+	if (!info) {
+		unsigned num_perfcounters =
+			r600_get_perfcounter_info(rscreen, 0, NULL);
+
+		return num_queries + num_perfcounters;
+	}
+
+	if (index >= num_queries)
+		return r600_get_perfcounter_info(rscreen, index - num_queries, info);
+
+	*info = r600_driver_query_list[index];
+
+	switch (info->query_type) {
+	case R600_QUERY_REQUESTED_VRAM:
+	case R600_QUERY_VRAM_USAGE:
+	case R600_QUERY_MAPPED_VRAM:
+		info->max_value.u64 = rscreen->info.vram_size;
+		break;
+	case R600_QUERY_REQUESTED_GTT:
+	case R600_QUERY_GTT_USAGE:
+	case R600_QUERY_MAPPED_GTT:
+		info->max_value.u64 = rscreen->info.gart_size;
+		break;
+	case R600_QUERY_GPU_TEMPERATURE:
+		info->max_value.u64 = 125;
+		break;
+	case R600_QUERY_VRAM_VIS_USAGE:
+		info->max_value.u64 = rscreen->info.vram_vis_size;
+		break;
+	}
+
+	if (info->group_id != ~(unsigned)0 && rscreen->perfcounters)
+		info->group_id += rscreen->perfcounters->num_groups;
+
+	return 1;
+}
+
+/* Note: Unfortunately, GPUPerfStudio hardcodes the order of hardware
+ * performance counter groups, so be careful when changing this and related
+ * functions.
+ */
+static int r600_get_driver_query_group_info(struct pipe_screen *screen,
+					    unsigned index,
+					    struct pipe_driver_query_group_info *info)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen *)screen;
+	unsigned num_pc_groups = 0;
+
+	if (rscreen->perfcounters)
+		num_pc_groups = rscreen->perfcounters->num_groups;
+
+	if (!info)
+		return num_pc_groups + R600_NUM_SW_QUERY_GROUPS;
+
+	if (index < num_pc_groups)
+		return r600_get_perfcounter_group_info(rscreen, index, info);
+
+	index -= num_pc_groups;
+	if (index >= R600_NUM_SW_QUERY_GROUPS)
+		return 0;
+
+	info->name = "GPIN";
+	info->max_active_queries = 5;
+	info->num_queries = 5;
+	return 1;
+}
+
+void r600_query_init(struct r600_common_context *rctx)
+{
+	rctx->b.create_query = r600_create_query;
+	rctx->b.create_batch_query = r600_create_batch_query;
+	rctx->b.destroy_query = r600_destroy_query;
+	rctx->b.begin_query = r600_begin_query;
+	rctx->b.end_query = r600_end_query;
+	rctx->b.get_query_result = r600_get_query_result;
+	rctx->b.get_query_result_resource = r600_get_query_result_resource;
+	rctx->render_cond_atom.emit = r600_emit_query_predication;
+
+	if (((struct r600_common_screen*)rctx->b.screen)->info.num_render_backends > 0)
+	    rctx->b.render_condition = r600_render_condition;
+
+	LIST_INITHEAD(&rctx->active_queries);
+}
+
+void r600_init_screen_query_functions(struct r600_common_screen *rscreen)
+{
+	rscreen->b.get_driver_query_info = r600_get_driver_query_info;
+	rscreen->b.get_driver_query_group_info = r600_get_driver_query_group_info;
+}
diff --git a/src/gallium/drivers/r600/r600_query.h b/src/gallium/drivers/r600/r600_query.h
new file mode 100644
index 00000000000..7455c8e63a8
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_query.h
@@ -0,0 +1,327 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *  Nicolai Hähnle <[email protected]>
+ *
+ */
+
+#ifndef R600_QUERY_H
+#define R600_QUERY_H
+
+#include "util/u_threaded_context.h"
+
+struct pipe_context;
+struct pipe_query;
+struct pipe_resource;
+
+struct r600_common_context;
+struct r600_common_screen;
+struct r600_query;
+struct r600_query_hw;
+struct r600_resource;
+
+enum {
+	R600_QUERY_DRAW_CALLS = PIPE_QUERY_DRIVER_SPECIFIC,
+	R600_QUERY_DECOMPRESS_CALLS,
+	R600_QUERY_MRT_DRAW_CALLS,
+	R600_QUERY_PRIM_RESTART_CALLS,
+	R600_QUERY_SPILL_DRAW_CALLS,
+	R600_QUERY_COMPUTE_CALLS,
+	R600_QUERY_SPILL_COMPUTE_CALLS,
+	R600_QUERY_DMA_CALLS,
+	R600_QUERY_CP_DMA_CALLS,
+	R600_QUERY_NUM_VS_FLUSHES,
+	R600_QUERY_NUM_PS_FLUSHES,
+	R600_QUERY_NUM_CS_FLUSHES,
+	R600_QUERY_NUM_CB_CACHE_FLUSHES,
+	R600_QUERY_NUM_DB_CACHE_FLUSHES,
+	R600_QUERY_NUM_L2_INVALIDATES,
+	R600_QUERY_NUM_L2_WRITEBACKS,
+	R600_QUERY_NUM_RESIDENT_HANDLES,
+	R600_QUERY_TC_OFFLOADED_SLOTS,
+	R600_QUERY_TC_DIRECT_SLOTS,
+	R600_QUERY_TC_NUM_SYNCS,
+	R600_QUERY_CS_THREAD_BUSY,
+	R600_QUERY_GALLIUM_THREAD_BUSY,
+	R600_QUERY_REQUESTED_VRAM,
+	R600_QUERY_REQUESTED_GTT,
+	R600_QUERY_MAPPED_VRAM,
+	R600_QUERY_MAPPED_GTT,
+	R600_QUERY_BUFFER_WAIT_TIME,
+	R600_QUERY_NUM_MAPPED_BUFFERS,
+	R600_QUERY_NUM_GFX_IBS,
+	R600_QUERY_NUM_SDMA_IBS,
+	R600_QUERY_GFX_BO_LIST_SIZE,
+	R600_QUERY_NUM_BYTES_MOVED,
+	R600_QUERY_NUM_EVICTIONS,
+	R600_QUERY_NUM_VRAM_CPU_PAGE_FAULTS,
+	R600_QUERY_VRAM_USAGE,
+	R600_QUERY_VRAM_VIS_USAGE,
+	R600_QUERY_GTT_USAGE,
+	R600_QUERY_GPU_TEMPERATURE,
+	R600_QUERY_CURRENT_GPU_SCLK,
+	R600_QUERY_CURRENT_GPU_MCLK,
+	R600_QUERY_GPU_LOAD,
+	R600_QUERY_GPU_SHADERS_BUSY,
+	R600_QUERY_GPU_TA_BUSY,
+	R600_QUERY_GPU_GDS_BUSY,
+	R600_QUERY_GPU_VGT_BUSY,
+	R600_QUERY_GPU_IA_BUSY,
+	R600_QUERY_GPU_SX_BUSY,
+	R600_QUERY_GPU_WD_BUSY,
+	R600_QUERY_GPU_BCI_BUSY,
+	R600_QUERY_GPU_SC_BUSY,
+	R600_QUERY_GPU_PA_BUSY,
+	R600_QUERY_GPU_DB_BUSY,
+	R600_QUERY_GPU_CP_BUSY,
+	R600_QUERY_GPU_CB_BUSY,
+	R600_QUERY_GPU_SDMA_BUSY,
+	R600_QUERY_GPU_PFP_BUSY,
+	R600_QUERY_GPU_MEQ_BUSY,
+	R600_QUERY_GPU_ME_BUSY,
+	R600_QUERY_GPU_SURF_SYNC_BUSY,
+	R600_QUERY_GPU_CP_DMA_BUSY,
+	R600_QUERY_GPU_SCRATCH_RAM_BUSY,
+	R600_QUERY_NUM_COMPILATIONS,
+	R600_QUERY_NUM_SHADERS_CREATED,
+	R600_QUERY_BACK_BUFFER_PS_DRAW_RATIO,
+	R600_QUERY_NUM_SHADER_CACHE_HITS,
+	R600_QUERY_GPIN_ASIC_ID,
+	R600_QUERY_GPIN_NUM_SIMD,
+	R600_QUERY_GPIN_NUM_RB,
+	R600_QUERY_GPIN_NUM_SPI,
+	R600_QUERY_GPIN_NUM_SE,
+
+	R600_QUERY_FIRST_PERFCOUNTER = PIPE_QUERY_DRIVER_SPECIFIC + 100,
+};
+
+enum {
+	R600_QUERY_GROUP_GPIN = 0,
+	R600_NUM_SW_QUERY_GROUPS
+};
+
+struct r600_query_ops {
+	void (*destroy)(struct r600_common_screen *, struct r600_query *);
+	bool (*begin)(struct r600_common_context *, struct r600_query *);
+	bool (*end)(struct r600_common_context *, struct r600_query *);
+	bool (*get_result)(struct r600_common_context *,
+			   struct r600_query *, bool wait,
+			   union pipe_query_result *result);
+	void (*get_result_resource)(struct r600_common_context *,
+				    struct r600_query *, bool wait,
+				    enum pipe_query_value_type result_type,
+				    int index,
+				    struct pipe_resource *resource,
+				    unsigned offset);
+};
+
+struct r600_query {
+	struct threaded_query b;
+	struct r600_query_ops *ops;
+
+	/* The type of query */
+	unsigned type;
+};
+
+enum {
+	R600_QUERY_HW_FLAG_NO_START = (1 << 0),
+	/* gap */
+	/* whether begin_query doesn't clear the result */
+	R600_QUERY_HW_FLAG_BEGIN_RESUMES = (1 << 2),
+};
+
+struct r600_query_hw_ops {
+	bool (*prepare_buffer)(struct r600_common_screen *,
+			       struct r600_query_hw *,
+			       struct r600_resource *);
+	void (*emit_start)(struct r600_common_context *,
+			   struct r600_query_hw *,
+			   struct r600_resource *buffer, uint64_t va);
+	void (*emit_stop)(struct r600_common_context *,
+			  struct r600_query_hw *,
+			  struct r600_resource *buffer, uint64_t va);
+	void (*clear_result)(struct r600_query_hw *, union pipe_query_result *);
+	void (*add_result)(struct r600_common_screen *screen,
+			   struct r600_query_hw *, void *buffer,
+			   union pipe_query_result *result);
+};
+
+struct r600_query_buffer {
+	/* The buffer where query results are stored. */
+	struct r600_resource		*buf;
+	/* Offset of the next free result after current query data */
+	unsigned			results_end;
+	/* If a query buffer is full, a new buffer is created and the old one
+	 * is put in here. When we calculate the result, we sum up the samples
+	 * from all buffers. */
+	struct r600_query_buffer	*previous;
+};
+
+struct r600_query_hw {
+	struct r600_query b;
+	struct r600_query_hw_ops *ops;
+	unsigned flags;
+
+	/* The query buffer and how many results are in it. */
+	struct r600_query_buffer buffer;
+	/* Size of the result in memory for both begin_query and end_query,
+	 * this can be one or two numbers, or it could even be a size of a structure. */
+	unsigned result_size;
+	/* The number of dwords for begin_query or end_query. */
+	unsigned num_cs_dw_begin;
+	unsigned num_cs_dw_end;
+	/* Linked list of queries */
+	struct list_head list;
+	/* For transform feedback: which stream the query is for */
+	unsigned stream;
+
+	/* Workaround via compute shader */
+	struct r600_resource *workaround_buf;
+	unsigned workaround_offset;
+};
+
+bool r600_query_hw_init(struct r600_common_screen *rscreen,
+			struct r600_query_hw *query);
+void r600_query_hw_destroy(struct r600_common_screen *rscreen,
+			   struct r600_query *rquery);
+bool r600_query_hw_begin(struct r600_common_context *rctx,
+			 struct r600_query *rquery);
+bool r600_query_hw_end(struct r600_common_context *rctx,
+		       struct r600_query *rquery);
+bool r600_query_hw_get_result(struct r600_common_context *rctx,
+			      struct r600_query *rquery,
+			      bool wait,
+			      union pipe_query_result *result);
+
+/* Performance counters */
+enum {
+	/* This block is part of the shader engine */
+	R600_PC_BLOCK_SE = (1 << 0),
+
+	/* Expose per-instance groups instead of summing all instances (within
+	 * an SE). */
+	R600_PC_BLOCK_INSTANCE_GROUPS = (1 << 1),
+
+	/* Expose per-SE groups instead of summing instances across SEs. */
+	R600_PC_BLOCK_SE_GROUPS = (1 << 2),
+
+	/* Shader block */
+	R600_PC_BLOCK_SHADER = (1 << 3),
+
+	/* Non-shader block with perfcounters windowed by shaders. */
+	R600_PC_BLOCK_SHADER_WINDOWED = (1 << 4),
+};
+
+/* Describes a hardware block with performance counters. Multiple instances of
+ * each block, possibly per-SE, may exist on the chip. Depending on the block
+ * and on the user's configuration, we either
+ *  (a) expose every instance as a performance counter group,
+ *  (b) expose a single performance counter group that reports the sum over all
+ *      instances, or
+ *  (c) expose one performance counter group per instance, but summed over all
+ *      shader engines.
+ */
+struct r600_perfcounter_block {
+	const char *basename;
+	unsigned flags;
+	unsigned num_counters;
+	unsigned num_selectors;
+	unsigned num_instances;
+
+	unsigned num_groups;
+	char *group_names;
+	unsigned group_name_stride;
+
+	char *selector_names;
+	unsigned selector_name_stride;
+
+	void *data;
+};
+
+struct r600_perfcounters {
+	unsigned num_groups;
+	unsigned num_blocks;
+	struct r600_perfcounter_block *blocks;
+
+	unsigned num_start_cs_dwords;
+	unsigned num_stop_cs_dwords;
+	unsigned num_instance_cs_dwords;
+	unsigned num_shaders_cs_dwords;
+
+	unsigned num_shader_types;
+	const char * const *shader_type_suffixes;
+	const unsigned *shader_type_bits;
+
+	void (*get_size)(struct r600_perfcounter_block *,
+			 unsigned count, unsigned *selectors,
+			 unsigned *num_select_dw, unsigned *num_read_dw);
+
+	void (*emit_instance)(struct r600_common_context *,
+			      int se, int instance);
+	void (*emit_shaders)(struct r600_common_context *, unsigned shaders);
+	void (*emit_select)(struct r600_common_context *,
+			    struct r600_perfcounter_block *,
+			    unsigned count, unsigned *selectors);
+	void (*emit_start)(struct r600_common_context *,
+			  struct r600_resource *buffer, uint64_t va);
+	void (*emit_stop)(struct r600_common_context *,
+			  struct r600_resource *buffer, uint64_t va);
+	void (*emit_read)(struct r600_common_context *,
+			  struct r600_perfcounter_block *,
+			  unsigned count, unsigned *selectors,
+			  struct r600_resource *buffer, uint64_t va);
+
+	void (*cleanup)(struct r600_common_screen *);
+
+	bool separate_se;
+	bool separate_instance;
+};
+
+struct pipe_query *r600_create_batch_query(struct pipe_context *ctx,
+					   unsigned num_queries,
+					   unsigned *query_types);
+
+int r600_get_perfcounter_info(struct r600_common_screen *,
+			      unsigned index,
+			      struct pipe_driver_query_info *info);
+int r600_get_perfcounter_group_info(struct r600_common_screen *,
+				    unsigned index,
+				    struct pipe_driver_query_group_info *info);
+
+bool r600_perfcounters_init(struct r600_perfcounters *, unsigned num_blocks);
+void r600_perfcounters_add_block(struct r600_common_screen *,
+				 struct r600_perfcounters *,
+				 const char *name, unsigned flags,
+				 unsigned counters, unsigned selectors,
+				 unsigned instances, void *data);
+void r600_perfcounters_do_destroy(struct r600_perfcounters *);
+void r600_query_hw_reset_buffers(struct r600_common_context *rctx,
+				 struct r600_query_hw *query);
+
+struct r600_qbo_state {
+	void *saved_compute;
+	struct pipe_constant_buffer saved_const0;
+	struct pipe_shader_buffer saved_ssbo[3];
+};
+
+#endif /* R600_QUERY_H */
diff --git a/src/gallium/drivers/r600/r600_streamout.c b/src/gallium/drivers/r600/r600_streamout.c
new file mode 100644
index 00000000000..a18089a3b39
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_streamout.c
@@ -0,0 +1,381 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors: Marek Olšák <[email protected]>
+ *
+ */
+
+#include "r600_pipe_common.h"
+#include "r600_cs.h"
+
+#include "util/u_memory.h"
+
+static void r600_set_streamout_enable(struct r600_common_context *rctx, bool enable);
+
+static struct pipe_stream_output_target *
+r600_create_so_target(struct pipe_context *ctx,
+		      struct pipe_resource *buffer,
+		      unsigned buffer_offset,
+		      unsigned buffer_size)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	struct r600_so_target *t;
+	struct r600_resource *rbuffer = (struct r600_resource*)buffer;
+
+	t = CALLOC_STRUCT(r600_so_target);
+	if (!t) {
+		return NULL;
+	}
+
+	u_suballocator_alloc(rctx->allocator_zeroed_memory, 4, 4,
+			     &t->buf_filled_size_offset,
+			     (struct pipe_resource**)&t->buf_filled_size);
+	if (!t->buf_filled_size) {
+		FREE(t);
+		return NULL;
+	}
+
+	t->b.reference.count = 1;
+	t->b.context = ctx;
+	pipe_resource_reference(&t->b.buffer, buffer);
+	t->b.buffer_offset = buffer_offset;
+	t->b.buffer_size = buffer_size;
+
+	util_range_add(&rbuffer->valid_buffer_range, buffer_offset,
+		       buffer_offset + buffer_size);
+	return &t->b;
+}
+
+static void r600_so_target_destroy(struct pipe_context *ctx,
+				   struct pipe_stream_output_target *target)
+{
+	struct r600_so_target *t = (struct r600_so_target*)target;
+	pipe_resource_reference(&t->b.buffer, NULL);
+	r600_resource_reference(&t->buf_filled_size, NULL);
+	FREE(t);
+}
+
+void r600_streamout_buffers_dirty(struct r600_common_context *rctx)
+{
+	struct r600_atom *begin = &rctx->streamout.begin_atom;
+	unsigned num_bufs = util_bitcount(rctx->streamout.enabled_mask);
+	unsigned num_bufs_appended = util_bitcount(rctx->streamout.enabled_mask &
+						   rctx->streamout.append_bitmask);
+
+	if (!num_bufs)
+		return;
+
+	rctx->streamout.num_dw_for_end =
+		12 + /* flush_vgt_streamout */
+		num_bufs * 11; /* STRMOUT_BUFFER_UPDATE, BUFFER_SIZE */
+
+	begin->num_dw = 12; /* flush_vgt_streamout */
+
+	if (rctx->chip_class >= SI) {
+		begin->num_dw += num_bufs * 4; /* SET_CONTEXT_REG */
+	} else {
+		begin->num_dw += num_bufs * 7; /* SET_CONTEXT_REG */
+
+		if (rctx->family >= CHIP_RS780 && rctx->family <= CHIP_RV740)
+			begin->num_dw += num_bufs * 5; /* STRMOUT_BASE_UPDATE */
+	}
+
+	begin->num_dw +=
+		num_bufs_appended * 8 + /* STRMOUT_BUFFER_UPDATE */
+		(num_bufs - num_bufs_appended) * 6 + /* STRMOUT_BUFFER_UPDATE */
+		(rctx->family > CHIP_R600 && rctx->family < CHIP_RS780 ? 2 : 0); /* SURFACE_BASE_UPDATE */
+
+	rctx->set_atom_dirty(rctx, begin, true);
+
+	r600_set_streamout_enable(rctx, true);
+}
+
+void r600_set_streamout_targets(struct pipe_context *ctx,
+				unsigned num_targets,
+				struct pipe_stream_output_target **targets,
+				const unsigned *offsets)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	unsigned i;
+        unsigned enabled_mask = 0, append_bitmask = 0;
+
+	/* Stop streamout. */
+	if (rctx->streamout.num_targets && rctx->streamout.begin_emitted) {
+		r600_emit_streamout_end(rctx);
+	}
+
+	/* Set the new targets. */
+	for (i = 0; i < num_targets; i++) {
+		pipe_so_target_reference((struct pipe_stream_output_target**)&rctx->streamout.targets[i], targets[i]);
+		if (!targets[i])
+			continue;
+
+		r600_context_add_resource_size(ctx, targets[i]->buffer);
+		enabled_mask |= 1 << i;
+		if (offsets[i] == ((unsigned)-1))
+			append_bitmask |= 1 << i;
+	}
+	for (; i < rctx->streamout.num_targets; i++) {
+		pipe_so_target_reference((struct pipe_stream_output_target**)&rctx->streamout.targets[i], NULL);
+	}
+
+	rctx->streamout.enabled_mask = enabled_mask;
+
+	rctx->streamout.num_targets = num_targets;
+	rctx->streamout.append_bitmask = append_bitmask;
+
+	if (num_targets) {
+		r600_streamout_buffers_dirty(rctx);
+	} else {
+		rctx->set_atom_dirty(rctx, &rctx->streamout.begin_atom, false);
+		r600_set_streamout_enable(rctx, false);
+	}
+}
+
+static void r600_flush_vgt_streamout(struct r600_common_context *rctx)
+{
+	struct radeon_winsys_cs *cs = rctx->gfx.cs;
+	unsigned reg_strmout_cntl;
+
+	/* The register is at different places on different ASICs. */
+	if (rctx->chip_class >= CIK) {
+		reg_strmout_cntl = R_0300FC_CP_STRMOUT_CNTL;
+	} else if (rctx->chip_class >= EVERGREEN) {
+		reg_strmout_cntl = R_0084FC_CP_STRMOUT_CNTL;
+	} else {
+		reg_strmout_cntl = R_008490_CP_STRMOUT_CNTL;
+	}
+
+	if (rctx->chip_class >= CIK) {
+		radeon_set_uconfig_reg(cs, reg_strmout_cntl, 0);
+	} else {
+		radeon_set_config_reg(cs, reg_strmout_cntl, 0);
+	}
+
+	radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
+	radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_SO_VGTSTREAMOUT_FLUSH) | EVENT_INDEX(0));
+
+	radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0));
+	radeon_emit(cs, WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */
+	radeon_emit(cs, reg_strmout_cntl >> 2);  /* register */
+	radeon_emit(cs, 0);
+	radeon_emit(cs, S_008490_OFFSET_UPDATE_DONE(1)); /* reference value */
+	radeon_emit(cs, S_008490_OFFSET_UPDATE_DONE(1)); /* mask */
+	radeon_emit(cs, 4); /* poll interval */
+}
+
+static void r600_emit_streamout_begin(struct r600_common_context *rctx, struct r600_atom *atom)
+{
+	struct radeon_winsys_cs *cs = rctx->gfx.cs;
+	struct r600_so_target **t = rctx->streamout.targets;
+	uint16_t *stride_in_dw = rctx->streamout.stride_in_dw;
+	unsigned i, update_flags = 0;
+
+	r600_flush_vgt_streamout(rctx);
+
+	for (i = 0; i < rctx->streamout.num_targets; i++) {
+		if (!t[i])
+			continue;
+
+		t[i]->stride_in_dw = stride_in_dw[i];
+
+		if (rctx->chip_class >= SI) {
+			/* SI binds streamout buffers as shader resources.
+			 * VGT only counts primitives and tells the shader
+			 * through SGPRs what to do. */
+			radeon_set_context_reg_seq(cs, R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16*i, 2);
+			radeon_emit(cs, (t[i]->b.buffer_offset +
+					 t[i]->b.buffer_size) >> 2);	/* BUFFER_SIZE (in DW) */
+			radeon_emit(cs, stride_in_dw[i]);		/* VTX_STRIDE (in DW) */
+		} else {
+			uint64_t va = r600_resource(t[i]->b.buffer)->gpu_address;
+
+			update_flags |= SURFACE_BASE_UPDATE_STRMOUT(i);
+
+			radeon_set_context_reg_seq(cs, R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16*i, 3);
+			radeon_emit(cs, (t[i]->b.buffer_offset +
+					 t[i]->b.buffer_size) >> 2);	/* BUFFER_SIZE (in DW) */
+			radeon_emit(cs, stride_in_dw[i]);		/* VTX_STRIDE (in DW) */
+			radeon_emit(cs, va >> 8);			/* BUFFER_BASE */
+
+			r600_emit_reloc(rctx, &rctx->gfx, r600_resource(t[i]->b.buffer),
+					RADEON_USAGE_WRITE, RADEON_PRIO_SHADER_RW_BUFFER);
+
+			/* R7xx requires this packet after updating BUFFER_BASE.
+			 * Without this, R7xx locks up. */
+			if (rctx->family >= CHIP_RS780 && rctx->family <= CHIP_RV740) {
+				radeon_emit(cs, PKT3(PKT3_STRMOUT_BASE_UPDATE, 1, 0));
+				radeon_emit(cs, i);
+				radeon_emit(cs, va >> 8);
+
+				r600_emit_reloc(rctx, &rctx->gfx, r600_resource(t[i]->b.buffer),
+						RADEON_USAGE_WRITE, RADEON_PRIO_SHADER_RW_BUFFER);
+			}
+		}
+
+		if (rctx->streamout.append_bitmask & (1 << i) && t[i]->buf_filled_size_valid) {
+			uint64_t va = t[i]->buf_filled_size->gpu_address +
+				      t[i]->buf_filled_size_offset;
+
+			/* Append. */
+			radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0));
+			radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) |
+				    STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_MEM)); /* control */
+			radeon_emit(cs, 0); /* unused */
+			radeon_emit(cs, 0); /* unused */
+			radeon_emit(cs, va); /* src address lo */
+			radeon_emit(cs, va >> 32); /* src address hi */
+
+			r600_emit_reloc(rctx,  &rctx->gfx, t[i]->buf_filled_size,
+					RADEON_USAGE_READ, RADEON_PRIO_SO_FILLED_SIZE);
+		} else {
+			/* Start from the beginning. */
+			radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0));
+			radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) |
+				    STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_PACKET)); /* control */
+			radeon_emit(cs, 0); /* unused */
+			radeon_emit(cs, 0); /* unused */
+			radeon_emit(cs, t[i]->b.buffer_offset >> 2); /* buffer offset in DW */
+			radeon_emit(cs, 0); /* unused */
+		}
+	}
+
+	if (rctx->family > CHIP_R600 && rctx->family < CHIP_RV770) {
+		radeon_emit(cs, PKT3(PKT3_SURFACE_BASE_UPDATE, 0, 0));
+		radeon_emit(cs, update_flags);
+	}
+	rctx->streamout.begin_emitted = true;
+}
+
+void r600_emit_streamout_end(struct r600_common_context *rctx)
+{
+	struct radeon_winsys_cs *cs = rctx->gfx.cs;
+	struct r600_so_target **t = rctx->streamout.targets;
+	unsigned i;
+	uint64_t va;
+
+	r600_flush_vgt_streamout(rctx);
+
+	for (i = 0; i < rctx->streamout.num_targets; i++) {
+		if (!t[i])
+			continue;
+
+		va = t[i]->buf_filled_size->gpu_address + t[i]->buf_filled_size_offset;
+		radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0));
+		radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) |
+			    STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_NONE) |
+			    STRMOUT_STORE_BUFFER_FILLED_SIZE); /* control */
+		radeon_emit(cs, va);     /* dst address lo */
+		radeon_emit(cs, va >> 32); /* dst address hi */
+		radeon_emit(cs, 0); /* unused */
+		radeon_emit(cs, 0); /* unused */
+
+		r600_emit_reloc(rctx,  &rctx->gfx, t[i]->buf_filled_size,
+				RADEON_USAGE_WRITE, RADEON_PRIO_SO_FILLED_SIZE);
+
+		/* Zero the buffer size. The counters (primitives generated,
+		 * primitives emitted) may be enabled even if there is not
+		 * buffer bound. This ensures that the primitives-emitted query
+		 * won't increment. */
+		radeon_set_context_reg(cs, R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16*i, 0);
+
+		t[i]->buf_filled_size_valid = true;
+	}
+
+	rctx->streamout.begin_emitted = false;
+	rctx->flags |= R600_CONTEXT_STREAMOUT_FLUSH;
+}
+
+/* STREAMOUT CONFIG DERIVED STATE
+ *
+ * Streamout must be enabled for the PRIMITIVES_GENERATED query to work.
+ * The buffer mask is an independent state, so no writes occur if there
+ * are no buffers bound.
+ */
+
+static void r600_emit_streamout_enable(struct r600_common_context *rctx,
+				       struct r600_atom *atom)
+{
+	unsigned strmout_config_reg = R_028AB0_VGT_STRMOUT_EN;
+	unsigned strmout_config_val = S_028B94_STREAMOUT_0_EN(r600_get_strmout_en(rctx));
+	unsigned strmout_buffer_reg = R_028B20_VGT_STRMOUT_BUFFER_EN;
+	unsigned strmout_buffer_val = rctx->streamout.hw_enabled_mask &
+				      rctx->streamout.enabled_stream_buffers_mask;
+
+	if (rctx->chip_class >= EVERGREEN) {
+		strmout_buffer_reg = R_028B98_VGT_STRMOUT_BUFFER_CONFIG;
+
+		strmout_config_reg = R_028B94_VGT_STRMOUT_CONFIG;
+		strmout_config_val |=
+			S_028B94_RAST_STREAM(0) |
+			S_028B94_STREAMOUT_1_EN(r600_get_strmout_en(rctx)) |
+			S_028B94_STREAMOUT_2_EN(r600_get_strmout_en(rctx)) |
+			S_028B94_STREAMOUT_3_EN(r600_get_strmout_en(rctx));
+	}
+	radeon_set_context_reg(rctx->gfx.cs, strmout_buffer_reg, strmout_buffer_val);
+	radeon_set_context_reg(rctx->gfx.cs, strmout_config_reg, strmout_config_val);
+}
+
+static void r600_set_streamout_enable(struct r600_common_context *rctx, bool enable)
+{
+	bool old_strmout_en = r600_get_strmout_en(rctx);
+	unsigned old_hw_enabled_mask = rctx->streamout.hw_enabled_mask;
+
+	rctx->streamout.streamout_enabled = enable;
+
+	rctx->streamout.hw_enabled_mask = rctx->streamout.enabled_mask |
+					  (rctx->streamout.enabled_mask << 4) |
+					  (rctx->streamout.enabled_mask << 8) |
+					  (rctx->streamout.enabled_mask << 12);
+
+	if ((old_strmout_en != r600_get_strmout_en(rctx)) ||
+            (old_hw_enabled_mask != rctx->streamout.hw_enabled_mask)) {
+		rctx->set_atom_dirty(rctx, &rctx->streamout.enable_atom, true);
+	}
+}
+
+void r600_update_prims_generated_query_state(struct r600_common_context *rctx,
+					     unsigned type, int diff)
+{
+	if (type == PIPE_QUERY_PRIMITIVES_GENERATED) {
+		bool old_strmout_en = r600_get_strmout_en(rctx);
+
+		rctx->streamout.num_prims_gen_queries += diff;
+		assert(rctx->streamout.num_prims_gen_queries >= 0);
+
+		rctx->streamout.prims_gen_query_enabled =
+			rctx->streamout.num_prims_gen_queries != 0;
+
+		if (old_strmout_en != r600_get_strmout_en(rctx)) {
+			rctx->set_atom_dirty(rctx, &rctx->streamout.enable_atom, true);
+		}
+	}
+}
+
+void r600_streamout_init(struct r600_common_context *rctx)
+{
+	rctx->b.create_stream_output_target = r600_create_so_target;
+	rctx->b.stream_output_target_destroy = r600_so_target_destroy;
+	rctx->streamout.begin_atom.emit = r600_emit_streamout_begin;
+	rctx->streamout.enable_atom.emit = r600_emit_streamout_enable;
+	rctx->streamout.enable_atom.num_dw = 6;
+}
diff --git a/src/gallium/drivers/r600/r600_test_dma.c b/src/gallium/drivers/r600/r600_test_dma.c
new file mode 100644
index 00000000000..9e1ff9e5fe0
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_test_dma.c
@@ -0,0 +1,398 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+/* This file implements randomized SDMA texture blit tests. */
+
+#include "r600_pipe_common.h"
+#include "util/u_surface.h"
+#include "util/rand_xor.h"
+
+static uint64_t seed_xorshift128plus[2];
+
+#define RAND_NUM_SIZE 8
+
+/* The GPU blits are emulated on the CPU using these CPU textures. */
+
+struct cpu_texture {
+	uint8_t *ptr;
+	uint64_t size;
+	uint64_t layer_stride;
+	unsigned stride;
+};
+
+static void alloc_cpu_texture(struct cpu_texture *tex,
+			      struct pipe_resource *templ, int bpp)
+{
+	tex->stride = align(templ->width0 * bpp, RAND_NUM_SIZE);
+	tex->layer_stride = (uint64_t)tex->stride * templ->height0;
+	tex->size = tex->layer_stride * templ->array_size;
+	tex->ptr = malloc(tex->size);
+	assert(tex->ptr);
+}
+
+static void set_random_pixels(struct pipe_context *ctx,
+			      struct pipe_resource *tex,
+			      struct cpu_texture *cpu)
+{
+	struct pipe_transfer *t;
+	uint8_t *map;
+	int x,y,z;
+
+	map = pipe_transfer_map_3d(ctx, tex, 0, PIPE_TRANSFER_WRITE,
+				   0, 0, 0, tex->width0, tex->height0,
+				   tex->array_size, &t);
+	assert(map);
+
+	for (z = 0; z < tex->array_size; z++) {
+		for (y = 0; y < tex->height0; y++) {
+			uint64_t *ptr = (uint64_t*)
+				(map + t->layer_stride*z + t->stride*y);
+			uint64_t *ptr_cpu = (uint64_t*)
+				(cpu->ptr + cpu->layer_stride*z + cpu->stride*y);
+			unsigned size = cpu->stride / RAND_NUM_SIZE;
+
+			assert(t->stride % RAND_NUM_SIZE == 0);
+			assert(cpu->stride % RAND_NUM_SIZE == 0);
+
+			for (x = 0; x < size; x++) {
+				*ptr++ = *ptr_cpu++ =
+					rand_xorshift128plus(seed_xorshift128plus);
+			}
+		}
+	}
+
+	pipe_transfer_unmap(ctx, t);
+}
+
+static bool compare_textures(struct pipe_context *ctx,
+			     struct pipe_resource *tex,
+			     struct cpu_texture *cpu, int bpp)
+{
+	struct pipe_transfer *t;
+	uint8_t *map;
+	int y,z;
+	bool pass = true;
+
+	map = pipe_transfer_map_3d(ctx, tex, 0, PIPE_TRANSFER_READ,
+				   0, 0, 0, tex->width0, tex->height0,
+				   tex->array_size, &t);
+	assert(map);
+
+	for (z = 0; z < tex->array_size; z++) {
+		for (y = 0; y < tex->height0; y++) {
+			uint8_t *ptr = map + t->layer_stride*z + t->stride*y;
+			uint8_t *cpu_ptr = cpu->ptr +
+					   cpu->layer_stride*z + cpu->stride*y;
+
+			if (memcmp(ptr, cpu_ptr, tex->width0 * bpp)) {
+				pass = false;
+				goto done;
+			}
+		}
+	}
+done:
+	pipe_transfer_unmap(ctx, t);
+	return pass;
+}
+
+static enum pipe_format get_format_from_bpp(int bpp)
+{
+	switch (bpp) {
+	case 1:
+		return PIPE_FORMAT_R8_UINT;
+	case 2:
+		return PIPE_FORMAT_R16_UINT;
+	case 4:
+		return PIPE_FORMAT_R32_UINT;
+	case 8:
+		return PIPE_FORMAT_R32G32_UINT;
+	case 16:
+		return PIPE_FORMAT_R32G32B32A32_UINT;
+	default:
+		assert(0);
+		return PIPE_FORMAT_NONE;
+	}
+}
+
+static const char *array_mode_to_string(struct r600_common_screen *rscreen,
+					struct radeon_surf *surf)
+{
+	if (rscreen->chip_class >= GFX9) {
+		/* TODO */
+		return "       UNKNOWN";
+	} else {
+		switch (surf->u.legacy.level[0].mode) {
+		case RADEON_SURF_MODE_LINEAR_ALIGNED:
+			return "LINEAR_ALIGNED";
+		case RADEON_SURF_MODE_1D:
+			return "1D_TILED_THIN1";
+		case RADEON_SURF_MODE_2D:
+			return "2D_TILED_THIN1";
+		default:
+			assert(0);
+			return "       UNKNOWN";
+		}
+	}
+}
+
+static unsigned generate_max_tex_side(unsigned max_tex_side)
+{
+	switch (rand() % 4) {
+	case 0:
+		/* Try to hit large sizes in 1/4 of the cases. */
+		return max_tex_side;
+	case 1:
+		/* Try to hit 1D tiling in 1/4 of the cases. */
+		return 128;
+	default:
+		/* Try to hit common sizes in 2/4 of the cases. */
+		return 2048;
+	}
+}
+
+void r600_test_dma(struct r600_common_screen *rscreen)
+{
+	struct pipe_screen *screen = &rscreen->b;
+	struct pipe_context *ctx = screen->context_create(screen, NULL, 0);
+	struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+	uint64_t max_alloc_size;
+	unsigned i, iterations, num_partial_copies, max_levels, max_tex_side;
+	unsigned num_pass = 0, num_fail = 0;
+
+	max_levels = screen->get_param(screen, PIPE_CAP_MAX_TEXTURE_2D_LEVELS);
+	max_tex_side = 1 << (max_levels - 1);
+
+	/* Max 128 MB allowed for both textures. */
+	max_alloc_size = 128 * 1024 * 1024;
+
+	/* the seed for random test parameters */
+	srand(0x9b47d95b);
+	/* the seed for random pixel data */
+	s_rand_xorshift128plus(seed_xorshift128plus, false);
+
+	iterations = 1000000000; /* just kill it when you are bored */
+	num_partial_copies = 30;
+
+	/* These parameters are randomly generated per test:
+	 * - whether to do one whole-surface copy or N partial copies per test
+	 * - which tiling modes to use (LINEAR_ALIGNED, 1D, 2D)
+	 * - which texture dimensions to use
+	 * - whether to use VRAM (all tiling modes) and GTT (staging, linear
+	 *   only) allocations
+	 * - random initial pixels in src
+	 * - generate random subrectangle copies for partial blits
+	 */
+	for (i = 0; i < iterations; i++) {
+		struct pipe_resource tsrc = {}, tdst = {}, *src, *dst;
+		struct r600_texture *rdst;
+		struct r600_texture *rsrc;
+		struct cpu_texture src_cpu, dst_cpu;
+		unsigned bpp, max_width, max_height, max_depth, j, num;
+		unsigned gfx_blits = 0, dma_blits = 0, max_tex_side_gen;
+		unsigned max_tex_layers;
+		bool pass;
+		bool do_partial_copies = rand() & 1;
+
+		/* generate a random test case */
+		tsrc.target = tdst.target = PIPE_TEXTURE_2D_ARRAY;
+		tsrc.depth0 = tdst.depth0 = 1;
+
+		bpp = 1 << (rand() % 5);
+		tsrc.format = tdst.format = get_format_from_bpp(bpp);
+
+		max_tex_side_gen = generate_max_tex_side(max_tex_side);
+		max_tex_layers = rand() % 4 ? 1 : 5;
+
+		tsrc.width0 = (rand() % max_tex_side_gen) + 1;
+		tsrc.height0 = (rand() % max_tex_side_gen) + 1;
+		tsrc.array_size = (rand() % max_tex_layers) + 1;
+
+		/* Have a 1/4 chance of getting power-of-two dimensions. */
+		if (rand() % 4 == 0) {
+			tsrc.width0 = util_next_power_of_two(tsrc.width0);
+			tsrc.height0 = util_next_power_of_two(tsrc.height0);
+		}
+
+		if (!do_partial_copies) {
+			/* whole-surface copies only, same dimensions */
+			tdst = tsrc;
+		} else {
+			max_tex_side_gen = generate_max_tex_side(max_tex_side);
+			max_tex_layers = rand() % 4 ? 1 : 5;
+
+			/* many partial copies, dimensions can be different */
+			tdst.width0 = (rand() % max_tex_side_gen) + 1;
+			tdst.height0 = (rand() % max_tex_side_gen) + 1;
+			tdst.array_size = (rand() % max_tex_layers) + 1;
+
+			/* Have a 1/4 chance of getting power-of-two dimensions. */
+			if (rand() % 4 == 0) {
+				tdst.width0 = util_next_power_of_two(tdst.width0);
+				tdst.height0 = util_next_power_of_two(tdst.height0);
+			}
+		}
+
+		/* check texture sizes */
+		if ((uint64_t)tsrc.width0 * tsrc.height0 * tsrc.array_size * bpp +
+		    (uint64_t)tdst.width0 * tdst.height0 * tdst.array_size * bpp >
+		    max_alloc_size) {
+			/* too large, try again */
+			i--;
+			continue;
+		}
+
+		/* VRAM + the tiling mode depends on dimensions (3/4 of cases),
+		 * or GTT + linear only (1/4 of cases)
+		 */
+		tsrc.usage = rand() % 4 ? PIPE_USAGE_DEFAULT : PIPE_USAGE_STAGING;
+		tdst.usage = rand() % 4 ? PIPE_USAGE_DEFAULT : PIPE_USAGE_STAGING;
+
+		/* Allocate textures (both the GPU and CPU copies).
+		 * The CPU will emulate what the GPU should be doing.
+		 */
+		src = screen->resource_create(screen, &tsrc);
+		dst = screen->resource_create(screen, &tdst);
+		assert(src);
+		assert(dst);
+		rdst = (struct r600_texture*)dst;
+		rsrc = (struct r600_texture*)src;
+		alloc_cpu_texture(&src_cpu, &tsrc, bpp);
+		alloc_cpu_texture(&dst_cpu, &tdst, bpp);
+
+		printf("%4u: dst = (%5u x %5u x %u, %s), "
+		       " src = (%5u x %5u x %u, %s), bpp = %2u, ",
+		       i, tdst.width0, tdst.height0, tdst.array_size,
+		       array_mode_to_string(rscreen, &rdst->surface),
+		       tsrc.width0, tsrc.height0, tsrc.array_size,
+		       array_mode_to_string(rscreen, &rsrc->surface), bpp);
+		fflush(stdout);
+
+		/* set src pixels */
+		set_random_pixels(ctx, src, &src_cpu);
+
+		/* clear dst pixels */
+		rctx->clear_buffer(ctx, dst, 0, rdst->surface.surf_size, 0, true);
+		memset(dst_cpu.ptr, 0, dst_cpu.layer_stride * tdst.array_size);
+
+		/* preparation */
+		max_width = MIN2(tsrc.width0, tdst.width0);
+		max_height = MIN2(tsrc.height0, tdst.height0);
+		max_depth = MIN2(tsrc.array_size, tdst.array_size);
+
+		num = do_partial_copies ? num_partial_copies : 1;
+		for (j = 0; j < num; j++) {
+			int width, height, depth;
+			int srcx, srcy, srcz, dstx, dsty, dstz;
+			struct pipe_box box;
+			unsigned old_num_draw_calls = rctx->num_draw_calls;
+			unsigned old_num_dma_calls = rctx->num_dma_calls;
+
+			if (!do_partial_copies) {
+				/* copy whole src to dst */
+				width = max_width;
+				height = max_height;
+				depth = max_depth;
+
+				srcx = srcy = srcz = dstx = dsty = dstz = 0;
+			} else {
+				/* random sub-rectangle copies from src to dst */
+				depth = (rand() % max_depth) + 1;
+				srcz = rand() % (tsrc.array_size - depth + 1);
+				dstz = rand() % (tdst.array_size - depth + 1);
+
+				/* special code path to hit the tiled partial copies */
+				if (!rsrc->surface.is_linear &&
+				    !rdst->surface.is_linear &&
+				    rand() & 1) {
+					if (max_width < 8 || max_height < 8)
+						continue;
+					width = ((rand() % (max_width / 8)) + 1) * 8;
+					height = ((rand() % (max_height / 8)) + 1) * 8;
+
+					srcx = rand() % (tsrc.width0 - width + 1) & ~0x7;
+					srcy = rand() % (tsrc.height0 - height + 1) & ~0x7;
+
+					dstx = rand() % (tdst.width0 - width + 1) & ~0x7;
+					dsty = rand() % (tdst.height0 - height + 1) & ~0x7;
+				} else {
+					/* just make sure that it doesn't divide by zero */
+					assert(max_width > 0 && max_height > 0);
+
+					width = (rand() % max_width) + 1;
+					height = (rand() % max_height) + 1;
+
+					srcx = rand() % (tsrc.width0 - width + 1);
+					srcy = rand() % (tsrc.height0 - height + 1);
+
+					dstx = rand() % (tdst.width0 - width + 1);
+					dsty = rand() % (tdst.height0 - height + 1);
+				}
+
+				/* special code path to hit out-of-bounds reads in L2T */
+				if (rsrc->surface.is_linear &&
+				    !rdst->surface.is_linear &&
+				    rand() % 4 == 0) {
+					srcx = 0;
+					srcy = 0;
+					srcz = 0;
+				}
+			}
+
+			/* GPU copy */
+			u_box_3d(srcx, srcy, srcz, width, height, depth, &box);
+			rctx->dma_copy(ctx, dst, 0, dstx, dsty, dstz, src, 0, &box);
+
+			/* See which engine was used. */
+			gfx_blits += rctx->num_draw_calls > old_num_draw_calls;
+			dma_blits += rctx->num_dma_calls > old_num_dma_calls;
+
+			/* CPU copy */
+			util_copy_box(dst_cpu.ptr, tdst.format, dst_cpu.stride,
+				      dst_cpu.layer_stride,
+				      dstx, dsty, dstz, width, height, depth,
+				      src_cpu.ptr, src_cpu.stride,
+				      src_cpu.layer_stride,
+				      srcx, srcy, srcz);
+		}
+
+		pass = compare_textures(ctx, dst, &dst_cpu, bpp);
+		if (pass)
+			num_pass++;
+		else
+			num_fail++;
+
+		printf("BLITs: GFX = %2u, DMA = %2u, %s [%u/%u]\n",
+		       gfx_blits, dma_blits, pass ? "pass" : "fail",
+		       num_pass, num_pass+num_fail);
+
+		/* cleanup */
+		pipe_resource_reference(&src, NULL);
+		pipe_resource_reference(&dst, NULL);
+		free(src_cpu.ptr);
+		free(dst_cpu.ptr);
+	}
+
+	ctx->destroy(ctx);
+	exit(0);
+}
diff --git a/src/gallium/drivers/r600/r600_texture.c b/src/gallium/drivers/r600/r600_texture.c
new file mode 100644
index 00000000000..15a1ec92675
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_texture.c
@@ -0,0 +1,2464 @@
+/*
+ * Copyright 2010 Jerome Glisse <[email protected]>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *      Jerome Glisse
+ *      Corbin Simpson
+ */
+#include "r600_pipe_common.h"
+#include "r600_cs.h"
+#include "r600_query.h"
+#include "util/u_format.h"
+#include "util/u_log.h"
+#include "util/u_memory.h"
+#include "util/u_pack_color.h"
+#include "util/u_surface.h"
+#include "os/os_time.h"
+#include <errno.h>
+#include <inttypes.h>
+
+static void r600_texture_discard_cmask(struct r600_common_screen *rscreen,
+				       struct r600_texture *rtex);
+static enum radeon_surf_mode
+r600_choose_tiling(struct r600_common_screen *rscreen,
+		   const struct pipe_resource *templ);
+
+
+bool r600_prepare_for_dma_blit(struct r600_common_context *rctx,
+			       struct r600_texture *rdst,
+			       unsigned dst_level, unsigned dstx,
+			       unsigned dsty, unsigned dstz,
+			       struct r600_texture *rsrc,
+			       unsigned src_level,
+			       const struct pipe_box *src_box)
+{
+	if (!rctx->dma.cs)
+		return false;
+
+	if (rdst->surface.bpe != rsrc->surface.bpe)
+		return false;
+
+	/* MSAA: Blits don't exist in the real world. */
+	if (rsrc->resource.b.b.nr_samples > 1 ||
+	    rdst->resource.b.b.nr_samples > 1)
+		return false;
+
+	/* Depth-stencil surfaces:
+	 *   When dst is linear, the DB->CB copy preserves HTILE.
+	 *   When dst is tiled, the 3D path must be used to update HTILE.
+	 */
+	if (rsrc->is_depth || rdst->is_depth)
+		return false;
+
+	/* CMASK as:
+	 *   src: Both texture and SDMA paths need decompression. Use SDMA.
+	 *   dst: If overwriting the whole texture, discard CMASK and use
+	 *        SDMA. Otherwise, use the 3D path.
+	 */
+	if (rdst->cmask.size && rdst->dirty_level_mask & (1 << dst_level)) {
+		/* The CMASK clear is only enabled for the first level. */
+		assert(dst_level == 0);
+		if (!util_texrange_covers_whole_level(&rdst->resource.b.b, dst_level,
+						      dstx, dsty, dstz, src_box->width,
+						      src_box->height, src_box->depth))
+			return false;
+
+		r600_texture_discard_cmask(rctx->screen, rdst);
+	}
+
+	/* All requirements are met. Prepare textures for SDMA. */
+	if (rsrc->cmask.size && rsrc->dirty_level_mask & (1 << src_level))
+		rctx->b.flush_resource(&rctx->b, &rsrc->resource.b.b);
+
+	assert(!(rsrc->dirty_level_mask & (1 << src_level)));
+	assert(!(rdst->dirty_level_mask & (1 << dst_level)));
+
+	return true;
+}
+
+/* Same as resource_copy_region, except that both upsampling and downsampling are allowed. */
+static void r600_copy_region_with_blit(struct pipe_context *pipe,
+				       struct pipe_resource *dst,
+                                       unsigned dst_level,
+                                       unsigned dstx, unsigned dsty, unsigned dstz,
+                                       struct pipe_resource *src,
+                                       unsigned src_level,
+                                       const struct pipe_box *src_box)
+{
+	struct pipe_blit_info blit;
+
+	memset(&blit, 0, sizeof(blit));
+	blit.src.resource = src;
+	blit.src.format = src->format;
+	blit.src.level = src_level;
+	blit.src.box = *src_box;
+	blit.dst.resource = dst;
+	blit.dst.format = dst->format;
+	blit.dst.level = dst_level;
+	blit.dst.box.x = dstx;
+	blit.dst.box.y = dsty;
+	blit.dst.box.z = dstz;
+	blit.dst.box.width = src_box->width;
+	blit.dst.box.height = src_box->height;
+	blit.dst.box.depth = src_box->depth;
+	blit.mask = util_format_get_mask(src->format) &
+		    util_format_get_mask(dst->format);
+	blit.filter = PIPE_TEX_FILTER_NEAREST;
+
+	if (blit.mask) {
+		pipe->blit(pipe, &blit);
+	}
+}
+
+/* Copy from a full GPU texture to a transfer's staging one. */
+static void r600_copy_to_staging_texture(struct pipe_context *ctx, struct r600_transfer *rtransfer)
+{
+	struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+	struct pipe_transfer *transfer = (struct pipe_transfer*)rtransfer;
+	struct pipe_resource *dst = &rtransfer->staging->b.b;
+	struct pipe_resource *src = transfer->resource;
+
+	if (src->nr_samples > 1) {
+		r600_copy_region_with_blit(ctx, dst, 0, 0, 0, 0,
+					   src, transfer->level, &transfer->box);
+		return;
+	}
+
+	rctx->dma_copy(ctx, dst, 0, 0, 0, 0, src, transfer->level,
+		       &transfer->box);
+}
+
+/* Copy from a transfer's staging texture to a full GPU one. */
+static void r600_copy_from_staging_texture(struct pipe_context *ctx, struct r600_transfer *rtransfer)
+{
+	struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+	struct pipe_transfer *transfer = (struct pipe_transfer*)rtransfer;
+	struct pipe_resource *dst = transfer->resource;
+	struct pipe_resource *src = &rtransfer->staging->b.b;
+	struct pipe_box sbox;
+
+	u_box_3d(0, 0, 0, transfer->box.width, transfer->box.height, transfer->box.depth, &sbox);
+
+	if (dst->nr_samples > 1) {
+		r600_copy_region_with_blit(ctx, dst, transfer->level,
+					   transfer->box.x, transfer->box.y, transfer->box.z,
+					   src, 0, &sbox);
+		return;
+	}
+
+	rctx->dma_copy(ctx, dst, transfer->level,
+		       transfer->box.x, transfer->box.y, transfer->box.z,
+		       src, 0, &sbox);
+}
+
+static unsigned r600_texture_get_offset(struct r600_common_screen *rscreen,
+					struct r600_texture *rtex, unsigned level,
+					const struct pipe_box *box,
+					unsigned *stride,
+					unsigned *layer_stride)
+{
+	if (rscreen->chip_class >= GFX9) {
+		*stride = rtex->surface.u.gfx9.surf_pitch * rtex->surface.bpe;
+		*layer_stride = rtex->surface.u.gfx9.surf_slice_size;
+
+		if (!box)
+			return 0;
+
+		/* Each texture is an array of slices. Each slice is an array
+		 * of mipmap levels. */
+		return box->z * rtex->surface.u.gfx9.surf_slice_size +
+		       rtex->surface.u.gfx9.offset[level] +
+		       (box->y / rtex->surface.blk_h *
+			rtex->surface.u.gfx9.surf_pitch +
+			box->x / rtex->surface.blk_w) * rtex->surface.bpe;
+	} else {
+		*stride = rtex->surface.u.legacy.level[level].nblk_x *
+			  rtex->surface.bpe;
+		*layer_stride = rtex->surface.u.legacy.level[level].slice_size;
+
+		if (!box)
+			return rtex->surface.u.legacy.level[level].offset;
+
+		/* Each texture is an array of mipmap levels. Each level is
+		 * an array of slices. */
+		return rtex->surface.u.legacy.level[level].offset +
+		       box->z * rtex->surface.u.legacy.level[level].slice_size +
+		       (box->y / rtex->surface.blk_h *
+		        rtex->surface.u.legacy.level[level].nblk_x +
+		        box->x / rtex->surface.blk_w) * rtex->surface.bpe;
+	}
+}
+
+static int r600_init_surface(struct r600_common_screen *rscreen,
+			     struct radeon_surf *surface,
+			     const struct pipe_resource *ptex,
+			     enum radeon_surf_mode array_mode,
+			     unsigned pitch_in_bytes_override,
+			     unsigned offset,
+			     bool is_imported,
+			     bool is_scanout,
+			     bool is_flushed_depth,
+			     bool tc_compatible_htile)
+{
+	const struct util_format_description *desc =
+		util_format_description(ptex->format);
+	bool is_depth, is_stencil;
+	int r;
+	unsigned i, bpe, flags = 0;
+
+	is_depth = util_format_has_depth(desc);
+	is_stencil = util_format_has_stencil(desc);
+
+	if (rscreen->chip_class >= EVERGREEN && !is_flushed_depth &&
+	    ptex->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
+		bpe = 4; /* stencil is allocated separately on evergreen */
+	} else {
+		bpe = util_format_get_blocksize(ptex->format);
+		assert(util_is_power_of_two(bpe));
+	}
+
+	if (!is_flushed_depth && is_depth) {
+		flags |= RADEON_SURF_ZBUFFER;
+
+		if (tc_compatible_htile &&
+		    (rscreen->chip_class >= GFX9 ||
+		     array_mode == RADEON_SURF_MODE_2D)) {
+			/* TC-compatible HTILE only supports Z32_FLOAT.
+			 * GFX9 also supports Z16_UNORM.
+			 * On VI, promote Z16 to Z32. DB->CB copies will convert
+			 * the format for transfers.
+			 */
+			if (rscreen->chip_class == VI)
+				bpe = 4;
+
+			flags |= RADEON_SURF_TC_COMPATIBLE_HTILE;
+		}
+
+		if (is_stencil)
+			flags |= RADEON_SURF_SBUFFER;
+	}
+
+	if (rscreen->chip_class >= VI &&
+	    (ptex->flags & R600_RESOURCE_FLAG_DISABLE_DCC ||
+	     ptex->format == PIPE_FORMAT_R9G9B9E5_FLOAT))
+		flags |= RADEON_SURF_DISABLE_DCC;
+
+	if (ptex->bind & PIPE_BIND_SCANOUT || is_scanout) {
+		/* This should catch bugs in gallium users setting incorrect flags. */
+		assert(ptex->nr_samples <= 1 &&
+		       ptex->array_size == 1 &&
+		       ptex->depth0 == 1 &&
+		       ptex->last_level == 0 &&
+		       !(flags & RADEON_SURF_Z_OR_SBUFFER));
+
+		flags |= RADEON_SURF_SCANOUT;
+	}
+
+	if (ptex->bind & PIPE_BIND_SHARED)
+		flags |= RADEON_SURF_SHAREABLE;
+	if (is_imported)
+		flags |= RADEON_SURF_IMPORTED | RADEON_SURF_SHAREABLE;
+	if (!(ptex->flags & R600_RESOURCE_FLAG_FORCE_TILING))
+		flags |= RADEON_SURF_OPTIMIZE_FOR_SPACE;
+
+	r = rscreen->ws->surface_init(rscreen->ws, ptex, flags, bpe,
+				      array_mode, surface);
+	if (r) {
+		return r;
+	}
+
+	if (rscreen->chip_class >= GFX9) {
+		assert(!pitch_in_bytes_override ||
+		       pitch_in_bytes_override == surface->u.gfx9.surf_pitch * bpe);
+		surface->u.gfx9.surf_offset = offset;
+	} else {
+		if (pitch_in_bytes_override &&
+		    pitch_in_bytes_override != surface->u.legacy.level[0].nblk_x * bpe) {
+			/* old ddx on evergreen over estimate alignment for 1d, only 1 level
+			 * for those
+			 */
+			surface->u.legacy.level[0].nblk_x = pitch_in_bytes_override / bpe;
+			surface->u.legacy.level[0].slice_size = pitch_in_bytes_override *
+								surface->u.legacy.level[0].nblk_y;
+		}
+
+		if (offset) {
+			for (i = 0; i < ARRAY_SIZE(surface->u.legacy.level); ++i)
+				surface->u.legacy.level[i].offset += offset;
+		}
+	}
+	return 0;
+}
+
+static void r600_texture_init_metadata(struct r600_common_screen *rscreen,
+				       struct r600_texture *rtex,
+				       struct radeon_bo_metadata *metadata)
+{
+	struct radeon_surf *surface = &rtex->surface;
+
+	memset(metadata, 0, sizeof(*metadata));
+
+	if (rscreen->chip_class >= GFX9) {
+		metadata->u.gfx9.swizzle_mode = surface->u.gfx9.surf.swizzle_mode;
+	} else {
+		metadata->u.legacy.microtile = surface->u.legacy.level[0].mode >= RADEON_SURF_MODE_1D ?
+					   RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
+		metadata->u.legacy.macrotile = surface->u.legacy.level[0].mode >= RADEON_SURF_MODE_2D ?
+					   RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
+		metadata->u.legacy.pipe_config = surface->u.legacy.pipe_config;
+		metadata->u.legacy.bankw = surface->u.legacy.bankw;
+		metadata->u.legacy.bankh = surface->u.legacy.bankh;
+		metadata->u.legacy.tile_split = surface->u.legacy.tile_split;
+		metadata->u.legacy.mtilea = surface->u.legacy.mtilea;
+		metadata->u.legacy.num_banks = surface->u.legacy.num_banks;
+		metadata->u.legacy.stride = surface->u.legacy.level[0].nblk_x * surface->bpe;
+		metadata->u.legacy.scanout = (surface->flags & RADEON_SURF_SCANOUT) != 0;
+	}
+}
+
+static void r600_surface_import_metadata(struct r600_common_screen *rscreen,
+					 struct radeon_surf *surf,
+					 struct radeon_bo_metadata *metadata,
+					 enum radeon_surf_mode *array_mode,
+					 bool *is_scanout)
+{
+	if (rscreen->chip_class >= GFX9) {
+		if (metadata->u.gfx9.swizzle_mode > 0)
+			*array_mode = RADEON_SURF_MODE_2D;
+		else
+			*array_mode = RADEON_SURF_MODE_LINEAR_ALIGNED;
+
+		*is_scanout = metadata->u.gfx9.swizzle_mode == 0 ||
+			      metadata->u.gfx9.swizzle_mode % 4 == 2;
+
+		surf->u.gfx9.surf.swizzle_mode = metadata->u.gfx9.swizzle_mode;
+	} else {
+		surf->u.legacy.pipe_config = metadata->u.legacy.pipe_config;
+		surf->u.legacy.bankw = metadata->u.legacy.bankw;
+		surf->u.legacy.bankh = metadata->u.legacy.bankh;
+		surf->u.legacy.tile_split = metadata->u.legacy.tile_split;
+		surf->u.legacy.mtilea = metadata->u.legacy.mtilea;
+		surf->u.legacy.num_banks = metadata->u.legacy.num_banks;
+
+		if (metadata->u.legacy.macrotile == RADEON_LAYOUT_TILED)
+			*array_mode = RADEON_SURF_MODE_2D;
+		else if (metadata->u.legacy.microtile == RADEON_LAYOUT_TILED)
+			*array_mode = RADEON_SURF_MODE_1D;
+		else
+			*array_mode = RADEON_SURF_MODE_LINEAR_ALIGNED;
+
+		*is_scanout = metadata->u.legacy.scanout;
+	}
+}
+
+static void r600_eliminate_fast_color_clear(struct r600_common_context *rctx,
+					    struct r600_texture *rtex)
+{
+	struct r600_common_screen *rscreen = rctx->screen;
+	struct pipe_context *ctx = &rctx->b;
+
+	if (ctx == rscreen->aux_context)
+		mtx_lock(&rscreen->aux_context_lock);
+
+	ctx->flush_resource(ctx, &rtex->resource.b.b);
+	ctx->flush(ctx, NULL, 0);
+
+	if (ctx == rscreen->aux_context)
+		mtx_unlock(&rscreen->aux_context_lock);
+}
+
+static void r600_texture_discard_cmask(struct r600_common_screen *rscreen,
+				       struct r600_texture *rtex)
+{
+	if (!rtex->cmask.size)
+		return;
+
+	assert(rtex->resource.b.b.nr_samples <= 1);
+
+	/* Disable CMASK. */
+	memset(&rtex->cmask, 0, sizeof(rtex->cmask));
+	rtex->cmask.base_address_reg = rtex->resource.gpu_address >> 8;
+	rtex->dirty_level_mask = 0;
+
+	if (rscreen->chip_class >= SI)
+		rtex->cb_color_info &= ~SI_S_028C70_FAST_CLEAR(1);
+	else
+		rtex->cb_color_info &= ~EG_S_028C70_FAST_CLEAR(1);
+
+	if (rtex->cmask_buffer != &rtex->resource)
+	    r600_resource_reference(&rtex->cmask_buffer, NULL);
+
+	/* Notify all contexts about the change. */
+	p_atomic_inc(&rscreen->dirty_tex_counter);
+	p_atomic_inc(&rscreen->compressed_colortex_counter);
+}
+
+static bool r600_can_disable_dcc(struct r600_texture *rtex)
+{
+	/* We can't disable DCC if it can be written by another process. */
+	return rtex->dcc_offset &&
+	       (!rtex->resource.b.is_shared ||
+		!(rtex->resource.external_usage & PIPE_HANDLE_USAGE_WRITE));
+}
+
+static bool r600_texture_discard_dcc(struct r600_common_screen *rscreen,
+				     struct r600_texture *rtex)
+{
+	if (!r600_can_disable_dcc(rtex))
+		return false;
+
+	assert(rtex->dcc_separate_buffer == NULL);
+
+	/* Disable DCC. */
+	rtex->dcc_offset = 0;
+
+	/* Notify all contexts about the change. */
+	p_atomic_inc(&rscreen->dirty_tex_counter);
+	return true;
+}
+
+/**
+ * Disable DCC for the texture. (first decompress, then discard metadata).
+ *
+ * There is unresolved multi-context synchronization issue between
+ * screen::aux_context and the current context. If applications do this with
+ * multiple contexts, it's already undefined behavior for them and we don't
+ * have to worry about that. The scenario is:
+ *
+ * If context 1 disables DCC and context 2 has queued commands that write
+ * to the texture via CB with DCC enabled, and the order of operations is
+ * as follows:
+ *   context 2 queues draw calls rendering to the texture, but doesn't flush
+ *   context 1 disables DCC and flushes
+ *   context 1 & 2 reset descriptors and FB state
+ *   context 2 flushes (new compressed tiles written by the draw calls)
+ *   context 1 & 2 read garbage, because DCC is disabled, yet there are
+ *   compressed tiled
+ *
+ * \param rctx  the current context if you have one, or rscreen->aux_context
+ *              if you don't.
+ */
+bool r600_texture_disable_dcc(struct r600_common_context *rctx,
+			      struct r600_texture *rtex)
+{
+	struct r600_common_screen *rscreen = rctx->screen;
+
+	if (!r600_can_disable_dcc(rtex))
+		return false;
+
+	if (&rctx->b == rscreen->aux_context)
+		mtx_lock(&rscreen->aux_context_lock);
+
+	/* Decompress DCC. */
+	rctx->decompress_dcc(&rctx->b, rtex);
+	rctx->b.flush(&rctx->b, NULL, 0);
+
+	if (&rctx->b == rscreen->aux_context)
+		mtx_unlock(&rscreen->aux_context_lock);
+
+	return r600_texture_discard_dcc(rscreen, rtex);
+}
+
+static void r600_reallocate_texture_inplace(struct r600_common_context *rctx,
+					    struct r600_texture *rtex,
+					    unsigned new_bind_flag,
+					    bool invalidate_storage)
+{
+	struct pipe_screen *screen = rctx->b.screen;
+	struct r600_texture *new_tex;
+	struct pipe_resource templ = rtex->resource.b.b;
+	unsigned i;
+
+	templ.bind |= new_bind_flag;
+
+	/* r600g doesn't react to dirty_tex_descriptor_counter */
+	if (rctx->chip_class < SI)
+		return;
+
+	if (rtex->resource.b.is_shared)
+		return;
+
+	if (new_bind_flag == PIPE_BIND_LINEAR) {
+		if (rtex->surface.is_linear)
+			return;
+
+		/* This fails with MSAA, depth, and compressed textures. */
+		if (r600_choose_tiling(rctx->screen, &templ) !=
+		    RADEON_SURF_MODE_LINEAR_ALIGNED)
+			return;
+	}
+
+	new_tex = (struct r600_texture*)screen->resource_create(screen, &templ);
+	if (!new_tex)
+		return;
+
+	/* Copy the pixels to the new texture. */
+	if (!invalidate_storage) {
+		for (i = 0; i <= templ.last_level; i++) {
+			struct pipe_box box;
+
+			u_box_3d(0, 0, 0,
+				 u_minify(templ.width0, i), u_minify(templ.height0, i),
+				 util_max_layer(&templ, i) + 1, &box);
+
+			rctx->dma_copy(&rctx->b, &new_tex->resource.b.b, i, 0, 0, 0,
+				       &rtex->resource.b.b, i, &box);
+		}
+	}
+
+	if (new_bind_flag == PIPE_BIND_LINEAR) {
+		r600_texture_discard_cmask(rctx->screen, rtex);
+		r600_texture_discard_dcc(rctx->screen, rtex);
+	}
+
+	/* Replace the structure fields of rtex. */
+	rtex->resource.b.b.bind = templ.bind;
+	pb_reference(&rtex->resource.buf, new_tex->resource.buf);
+	rtex->resource.gpu_address = new_tex->resource.gpu_address;
+	rtex->resource.vram_usage = new_tex->resource.vram_usage;
+	rtex->resource.gart_usage = new_tex->resource.gart_usage;
+	rtex->resource.bo_size = new_tex->resource.bo_size;
+	rtex->resource.bo_alignment = new_tex->resource.bo_alignment;
+	rtex->resource.domains = new_tex->resource.domains;
+	rtex->resource.flags = new_tex->resource.flags;
+	rtex->size = new_tex->size;
+	rtex->db_render_format = new_tex->db_render_format;
+	rtex->db_compatible = new_tex->db_compatible;
+	rtex->can_sample_z = new_tex->can_sample_z;
+	rtex->can_sample_s = new_tex->can_sample_s;
+	rtex->surface = new_tex->surface;
+	rtex->fmask = new_tex->fmask;
+	rtex->cmask = new_tex->cmask;
+	rtex->cb_color_info = new_tex->cb_color_info;
+	rtex->last_msaa_resolve_target_micro_mode = new_tex->last_msaa_resolve_target_micro_mode;
+	rtex->htile_offset = new_tex->htile_offset;
+	rtex->tc_compatible_htile = new_tex->tc_compatible_htile;
+	rtex->depth_cleared = new_tex->depth_cleared;
+	rtex->stencil_cleared = new_tex->stencil_cleared;
+	rtex->non_disp_tiling = new_tex->non_disp_tiling;
+	rtex->dcc_gather_statistics = new_tex->dcc_gather_statistics;
+	rtex->framebuffers_bound = new_tex->framebuffers_bound;
+
+	if (new_bind_flag == PIPE_BIND_LINEAR) {
+		assert(!rtex->htile_offset);
+		assert(!rtex->cmask.size);
+		assert(!rtex->fmask.size);
+		assert(!rtex->dcc_offset);
+		assert(!rtex->is_depth);
+	}
+
+	r600_texture_reference(&new_tex, NULL);
+
+	p_atomic_inc(&rctx->screen->dirty_tex_counter);
+}
+
+static boolean r600_texture_get_handle(struct pipe_screen* screen,
+				       struct pipe_context *ctx,
+				       struct pipe_resource *resource,
+				       struct winsys_handle *whandle,
+                                       unsigned usage)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+	struct r600_common_context *rctx;
+	struct r600_resource *res = (struct r600_resource*)resource;
+	struct r600_texture *rtex = (struct r600_texture*)resource;
+	struct radeon_bo_metadata metadata;
+	bool update_metadata = false;
+	unsigned stride, offset, slice_size;
+
+	ctx = threaded_context_unwrap_sync(ctx);
+	rctx = (struct r600_common_context*)(ctx ? ctx : rscreen->aux_context);
+
+	if (resource->target != PIPE_BUFFER) {
+		/* This is not supported now, but it might be required for OpenCL
+		 * interop in the future.
+		 */
+		if (resource->nr_samples > 1 || rtex->is_depth)
+			return false;
+
+		/* Move a suballocated texture into a non-suballocated allocation. */
+		if (rscreen->ws->buffer_is_suballocated(res->buf) ||
+		    rtex->surface.tile_swizzle) {
+			assert(!res->b.is_shared);
+			r600_reallocate_texture_inplace(rctx, rtex,
+							PIPE_BIND_SHARED, false);
+			rctx->b.flush(&rctx->b, NULL, 0);
+			assert(res->b.b.bind & PIPE_BIND_SHARED);
+			assert(res->flags & RADEON_FLAG_NO_SUBALLOC);
+			assert(rtex->surface.tile_swizzle == 0);
+		}
+
+		/* Since shader image stores don't support DCC on VI,
+		 * disable it for external clients that want write
+		 * access.
+		 */
+		if (usage & PIPE_HANDLE_USAGE_WRITE && rtex->dcc_offset) {
+			if (r600_texture_disable_dcc(rctx, rtex))
+				update_metadata = true;
+		}
+
+		if (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
+		    (rtex->cmask.size || rtex->dcc_offset)) {
+			/* Eliminate fast clear (both CMASK and DCC) */
+			r600_eliminate_fast_color_clear(rctx, rtex);
+
+			/* Disable CMASK if flush_resource isn't going
+			 * to be called.
+			 */
+			if (rtex->cmask.size)
+				r600_texture_discard_cmask(rscreen, rtex);
+		}
+
+		/* Set metadata. */
+		if (!res->b.is_shared || update_metadata) {
+			r600_texture_init_metadata(rscreen, rtex, &metadata);
+			if (rscreen->query_opaque_metadata)
+				rscreen->query_opaque_metadata(rscreen, rtex,
+							       &metadata);
+
+			rscreen->ws->buffer_set_metadata(res->buf, &metadata);
+		}
+
+		if (rscreen->chip_class >= GFX9) {
+			offset = rtex->surface.u.gfx9.surf_offset;
+			stride = rtex->surface.u.gfx9.surf_pitch *
+				 rtex->surface.bpe;
+			slice_size = rtex->surface.u.gfx9.surf_slice_size;
+		} else {
+			offset = rtex->surface.u.legacy.level[0].offset;
+			stride = rtex->surface.u.legacy.level[0].nblk_x *
+				 rtex->surface.bpe;
+			slice_size = rtex->surface.u.legacy.level[0].slice_size;
+		}
+	} else {
+		/* Move a suballocated buffer into a non-suballocated allocation. */
+		if (rscreen->ws->buffer_is_suballocated(res->buf)) {
+			assert(!res->b.is_shared);
+
+			/* Allocate a new buffer with PIPE_BIND_SHARED. */
+			struct pipe_resource templ = res->b.b;
+			templ.bind |= PIPE_BIND_SHARED;
+
+			struct pipe_resource *newb =
+				screen->resource_create(screen, &templ);
+			if (!newb)
+				return false;
+
+			/* Copy the old buffer contents to the new one. */
+			struct pipe_box box;
+			u_box_1d(0, newb->width0, &box);
+			rctx->b.resource_copy_region(&rctx->b, newb, 0, 0, 0, 0,
+						     &res->b.b, 0, &box);
+			/* Move the new buffer storage to the old pipe_resource. */
+			r600_replace_buffer_storage(&rctx->b, &res->b.b, newb);
+			pipe_resource_reference(&newb, NULL);
+
+			assert(res->b.b.bind & PIPE_BIND_SHARED);
+			assert(res->flags & RADEON_FLAG_NO_SUBALLOC);
+		}
+
+		/* Buffers */
+		offset = 0;
+		stride = 0;
+		slice_size = 0;
+	}
+
+	if (res->b.is_shared) {
+		/* USAGE_EXPLICIT_FLUSH must be cleared if at least one user
+		 * doesn't set it.
+		 */
+		res->external_usage |= usage & ~PIPE_HANDLE_USAGE_EXPLICIT_FLUSH;
+		if (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH))
+			res->external_usage &= ~PIPE_HANDLE_USAGE_EXPLICIT_FLUSH;
+	} else {
+		res->b.is_shared = true;
+		res->external_usage = usage;
+	}
+
+	return rscreen->ws->buffer_get_handle(res->buf, stride, offset,
+					      slice_size, whandle);
+}
+
+static void r600_texture_destroy(struct pipe_screen *screen,
+				 struct pipe_resource *ptex)
+{
+	struct r600_texture *rtex = (struct r600_texture*)ptex;
+	struct r600_resource *resource = &rtex->resource;
+
+	r600_texture_reference(&rtex->flushed_depth_texture, NULL);
+
+	if (rtex->cmask_buffer != &rtex->resource) {
+	    r600_resource_reference(&rtex->cmask_buffer, NULL);
+	}
+	pb_reference(&resource->buf, NULL);
+	r600_resource_reference(&rtex->dcc_separate_buffer, NULL);
+	r600_resource_reference(&rtex->last_dcc_separate_buffer, NULL);
+	FREE(rtex);
+}
+
+static const struct u_resource_vtbl r600_texture_vtbl;
+
+/* The number of samples can be specified independently of the texture. */
+void r600_texture_get_fmask_info(struct r600_common_screen *rscreen,
+				 struct r600_texture *rtex,
+				 unsigned nr_samples,
+				 struct r600_fmask_info *out)
+{
+	/* FMASK is allocated like an ordinary texture. */
+	struct pipe_resource templ = rtex->resource.b.b;
+	struct radeon_surf fmask = {};
+	unsigned flags, bpe;
+
+	memset(out, 0, sizeof(*out));
+
+	if (rscreen->chip_class >= GFX9) {
+		out->alignment = rtex->surface.u.gfx9.fmask_alignment;
+		out->size = rtex->surface.u.gfx9.fmask_size;
+		return;
+	}
+
+	templ.nr_samples = 1;
+	flags = rtex->surface.flags | RADEON_SURF_FMASK;
+
+	if (rscreen->chip_class <= CAYMAN) {
+		/* Use the same parameters and tile mode. */
+		fmask.u.legacy.bankw = rtex->surface.u.legacy.bankw;
+		fmask.u.legacy.bankh = rtex->surface.u.legacy.bankh;
+		fmask.u.legacy.mtilea = rtex->surface.u.legacy.mtilea;
+		fmask.u.legacy.tile_split = rtex->surface.u.legacy.tile_split;
+
+		if (nr_samples <= 4)
+			fmask.u.legacy.bankh = 4;
+	}
+
+	switch (nr_samples) {
+	case 2:
+	case 4:
+		bpe = 1;
+		break;
+	case 8:
+		bpe = 4;
+		break;
+	default:
+		R600_ERR("Invalid sample count for FMASK allocation.\n");
+		return;
+	}
+
+	/* Overallocate FMASK on R600-R700 to fix colorbuffer corruption.
+	 * This can be fixed by writing a separate FMASK allocator specifically
+	 * for R600-R700 asics. */
+	if (rscreen->chip_class <= R700) {
+		bpe *= 2;
+	}
+
+	if (rscreen->ws->surface_init(rscreen->ws, &templ, flags, bpe,
+				      RADEON_SURF_MODE_2D, &fmask)) {
+		R600_ERR("Got error in surface_init while allocating FMASK.\n");
+		return;
+	}
+
+	assert(fmask.u.legacy.level[0].mode == RADEON_SURF_MODE_2D);
+
+	out->slice_tile_max = (fmask.u.legacy.level[0].nblk_x * fmask.u.legacy.level[0].nblk_y) / 64;
+	if (out->slice_tile_max)
+		out->slice_tile_max -= 1;
+
+	out->tile_mode_index = fmask.u.legacy.tiling_index[0];
+	out->pitch_in_pixels = fmask.u.legacy.level[0].nblk_x;
+	out->bank_height = fmask.u.legacy.bankh;
+	out->tile_swizzle = fmask.tile_swizzle;
+	out->alignment = MAX2(256, fmask.surf_alignment);
+	out->size = fmask.surf_size;
+}
+
+static void r600_texture_allocate_fmask(struct r600_common_screen *rscreen,
+					struct r600_texture *rtex)
+{
+	r600_texture_get_fmask_info(rscreen, rtex,
+				    rtex->resource.b.b.nr_samples, &rtex->fmask);
+
+	rtex->fmask.offset = align64(rtex->size, rtex->fmask.alignment);
+	rtex->size = rtex->fmask.offset + rtex->fmask.size;
+}
+
+void r600_texture_get_cmask_info(struct r600_common_screen *rscreen,
+				 struct r600_texture *rtex,
+				 struct r600_cmask_info *out)
+{
+	unsigned cmask_tile_width = 8;
+	unsigned cmask_tile_height = 8;
+	unsigned cmask_tile_elements = cmask_tile_width * cmask_tile_height;
+	unsigned element_bits = 4;
+	unsigned cmask_cache_bits = 1024;
+	unsigned num_pipes = rscreen->info.num_tile_pipes;
+	unsigned pipe_interleave_bytes = rscreen->info.pipe_interleave_bytes;
+
+	unsigned elements_per_macro_tile = (cmask_cache_bits / element_bits) * num_pipes;
+	unsigned pixels_per_macro_tile = elements_per_macro_tile * cmask_tile_elements;
+	unsigned sqrt_pixels_per_macro_tile = sqrt(pixels_per_macro_tile);
+	unsigned macro_tile_width = util_next_power_of_two(sqrt_pixels_per_macro_tile);
+	unsigned macro_tile_height = pixels_per_macro_tile / macro_tile_width;
+
+	unsigned pitch_elements = align(rtex->resource.b.b.width0, macro_tile_width);
+	unsigned height = align(rtex->resource.b.b.height0, macro_tile_height);
+
+	unsigned base_align = num_pipes * pipe_interleave_bytes;
+	unsigned slice_bytes =
+		((pitch_elements * height * element_bits + 7) / 8) / cmask_tile_elements;
+
+	assert(macro_tile_width % 128 == 0);
+	assert(macro_tile_height % 128 == 0);
+
+	out->slice_tile_max = ((pitch_elements * height) / (128*128)) - 1;
+	out->alignment = MAX2(256, base_align);
+	out->size = (util_max_layer(&rtex->resource.b.b, 0) + 1) *
+		    align(slice_bytes, base_align);
+}
+
+static void si_texture_get_cmask_info(struct r600_common_screen *rscreen,
+				      struct r600_texture *rtex,
+				      struct r600_cmask_info *out)
+{
+	unsigned pipe_interleave_bytes = rscreen->info.pipe_interleave_bytes;
+	unsigned num_pipes = rscreen->info.num_tile_pipes;
+	unsigned cl_width, cl_height;
+
+	if (rscreen->chip_class >= GFX9) {
+		out->alignment = rtex->surface.u.gfx9.cmask_alignment;
+		out->size = rtex->surface.u.gfx9.cmask_size;
+		return;
+	}
+
+	switch (num_pipes) {
+	case 2:
+		cl_width = 32;
+		cl_height = 16;
+		break;
+	case 4:
+		cl_width = 32;
+		cl_height = 32;
+		break;
+	case 8:
+		cl_width = 64;
+		cl_height = 32;
+		break;
+	case 16: /* Hawaii */
+		cl_width = 64;
+		cl_height = 64;
+		break;
+	default:
+		assert(0);
+		return;
+	}
+
+	unsigned base_align = num_pipes * pipe_interleave_bytes;
+
+	unsigned width = align(rtex->resource.b.b.width0, cl_width*8);
+	unsigned height = align(rtex->resource.b.b.height0, cl_height*8);
+	unsigned slice_elements = (width * height) / (8*8);
+
+	/* Each element of CMASK is a nibble. */
+	unsigned slice_bytes = slice_elements / 2;
+
+	out->slice_tile_max = (width * height) / (128*128);
+	if (out->slice_tile_max)
+		out->slice_tile_max -= 1;
+
+	out->alignment = MAX2(256, base_align);
+	out->size = (util_max_layer(&rtex->resource.b.b, 0) + 1) *
+		    align(slice_bytes, base_align);
+}
+
+static void r600_texture_allocate_cmask(struct r600_common_screen *rscreen,
+					struct r600_texture *rtex)
+{
+	if (rscreen->chip_class >= SI) {
+		si_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
+	} else {
+		r600_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
+	}
+
+	rtex->cmask.offset = align64(rtex->size, rtex->cmask.alignment);
+	rtex->size = rtex->cmask.offset + rtex->cmask.size;
+
+	if (rscreen->chip_class >= SI)
+		rtex->cb_color_info |= SI_S_028C70_FAST_CLEAR(1);
+	else
+		rtex->cb_color_info |= EG_S_028C70_FAST_CLEAR(1);
+}
+
+static void r600_texture_alloc_cmask_separate(struct r600_common_screen *rscreen,
+					      struct r600_texture *rtex)
+{
+	if (rtex->cmask_buffer)
+                return;
+
+	assert(rtex->cmask.size == 0);
+
+	if (rscreen->chip_class >= SI) {
+		si_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
+	} else {
+		r600_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
+	}
+
+	rtex->cmask_buffer = (struct r600_resource *)
+		r600_aligned_buffer_create(&rscreen->b,
+					   R600_RESOURCE_FLAG_UNMAPPABLE,
+					   PIPE_USAGE_DEFAULT,
+					   rtex->cmask.size,
+					   rtex->cmask.alignment);
+	if (rtex->cmask_buffer == NULL) {
+		rtex->cmask.size = 0;
+		return;
+	}
+
+	/* update colorbuffer state bits */
+	rtex->cmask.base_address_reg = rtex->cmask_buffer->gpu_address >> 8;
+
+	if (rscreen->chip_class >= SI)
+		rtex->cb_color_info |= SI_S_028C70_FAST_CLEAR(1);
+	else
+		rtex->cb_color_info |= EG_S_028C70_FAST_CLEAR(1);
+
+	p_atomic_inc(&rscreen->compressed_colortex_counter);
+}
+
+static void r600_texture_get_htile_size(struct r600_common_screen *rscreen,
+					struct r600_texture *rtex)
+{
+	unsigned cl_width, cl_height, width, height;
+	unsigned slice_elements, slice_bytes, pipe_interleave_bytes, base_align;
+	unsigned num_pipes = rscreen->info.num_tile_pipes;
+
+	assert(rscreen->chip_class <= VI);
+
+	rtex->surface.htile_size = 0;
+
+	if (rscreen->chip_class <= EVERGREEN &&
+	    rscreen->info.drm_major == 2 && rscreen->info.drm_minor < 26)
+		return;
+
+	/* HW bug on R6xx. */
+	if (rscreen->chip_class == R600 &&
+	    (rtex->resource.b.b.width0 > 7680 ||
+	     rtex->resource.b.b.height0 > 7680))
+		return;
+
+	/* HTILE is broken with 1D tiling on old kernels and CIK. */
+	if (rscreen->chip_class >= CIK &&
+	    rtex->surface.u.legacy.level[0].mode == RADEON_SURF_MODE_1D &&
+	    rscreen->info.drm_major == 2 && rscreen->info.drm_minor < 38)
+		return;
+
+	/* Overalign HTILE on P2 configs to work around GPU hangs in
+	 * piglit/depthstencil-render-miplevels 585.
+	 *
+	 * This has been confirmed to help Kabini & Stoney, where the hangs
+	 * are always reproducible. I think I have seen the test hang
+	 * on Carrizo too, though it was very rare there.
+	 */
+	if (rscreen->chip_class >= CIK && num_pipes < 4)
+		num_pipes = 4;
+
+	switch (num_pipes) {
+	case 1:
+		cl_width = 32;
+		cl_height = 16;
+		break;
+	case 2:
+		cl_width = 32;
+		cl_height = 32;
+		break;
+	case 4:
+		cl_width = 64;
+		cl_height = 32;
+		break;
+	case 8:
+		cl_width = 64;
+		cl_height = 64;
+		break;
+	case 16:
+		cl_width = 128;
+		cl_height = 64;
+		break;
+	default:
+		assert(0);
+		return;
+	}
+
+	width = align(rtex->resource.b.b.width0, cl_width * 8);
+	height = align(rtex->resource.b.b.height0, cl_height * 8);
+
+	slice_elements = (width * height) / (8 * 8);
+	slice_bytes = slice_elements * 4;
+
+	pipe_interleave_bytes = rscreen->info.pipe_interleave_bytes;
+	base_align = num_pipes * pipe_interleave_bytes;
+
+	rtex->surface.htile_alignment = base_align;
+	rtex->surface.htile_size =
+		(util_max_layer(&rtex->resource.b.b, 0) + 1) *
+		align(slice_bytes, base_align);
+}
+
+static void r600_texture_allocate_htile(struct r600_common_screen *rscreen,
+					struct r600_texture *rtex)
+{
+	if (rscreen->chip_class <= VI && !rtex->tc_compatible_htile)
+		r600_texture_get_htile_size(rscreen, rtex);
+
+	if (!rtex->surface.htile_size)
+		return;
+
+	rtex->htile_offset = align(rtex->size, rtex->surface.htile_alignment);
+	rtex->size = rtex->htile_offset + rtex->surface.htile_size;
+}
+
+void r600_print_texture_info(struct r600_common_screen *rscreen,
+			     struct r600_texture *rtex, struct u_log_context *log)
+{
+	int i;
+
+	/* Common parameters. */
+	u_log_printf(log, "  Info: npix_x=%u, npix_y=%u, npix_z=%u, blk_w=%u, "
+		"blk_h=%u, array_size=%u, last_level=%u, "
+		"bpe=%u, nsamples=%u, flags=0x%x, %s\n",
+		rtex->resource.b.b.width0, rtex->resource.b.b.height0,
+		rtex->resource.b.b.depth0, rtex->surface.blk_w,
+		rtex->surface.blk_h,
+		rtex->resource.b.b.array_size, rtex->resource.b.b.last_level,
+		rtex->surface.bpe, rtex->resource.b.b.nr_samples,
+		rtex->surface.flags, util_format_short_name(rtex->resource.b.b.format));
+
+	if (rscreen->chip_class >= GFX9) {
+		u_log_printf(log, "  Surf: size=%"PRIu64", slice_size=%"PRIu64", "
+			"alignment=%u, swmode=%u, epitch=%u, pitch=%u\n",
+			rtex->surface.surf_size,
+			rtex->surface.u.gfx9.surf_slice_size,
+			rtex->surface.surf_alignment,
+			rtex->surface.u.gfx9.surf.swizzle_mode,
+			rtex->surface.u.gfx9.surf.epitch,
+			rtex->surface.u.gfx9.surf_pitch);
+
+		if (rtex->fmask.size) {
+			u_log_printf(log, "  FMASK: offset=%"PRIu64", size=%"PRIu64", "
+				"alignment=%u, swmode=%u, epitch=%u\n",
+				rtex->fmask.offset,
+				rtex->surface.u.gfx9.fmask_size,
+				rtex->surface.u.gfx9.fmask_alignment,
+				rtex->surface.u.gfx9.fmask.swizzle_mode,
+				rtex->surface.u.gfx9.fmask.epitch);
+		}
+
+		if (rtex->cmask.size) {
+			u_log_printf(log, "  CMask: offset=%"PRIu64", size=%"PRIu64", "
+				"alignment=%u, rb_aligned=%u, pipe_aligned=%u\n",
+				rtex->cmask.offset,
+				rtex->surface.u.gfx9.cmask_size,
+				rtex->surface.u.gfx9.cmask_alignment,
+				rtex->surface.u.gfx9.cmask.rb_aligned,
+				rtex->surface.u.gfx9.cmask.pipe_aligned);
+		}
+
+		if (rtex->htile_offset) {
+			u_log_printf(log, "  HTile: offset=%"PRIu64", size=%"PRIu64", alignment=%u, "
+				"rb_aligned=%u, pipe_aligned=%u\n",
+				rtex->htile_offset,
+				rtex->surface.htile_size,
+				rtex->surface.htile_alignment,
+				rtex->surface.u.gfx9.htile.rb_aligned,
+				rtex->surface.u.gfx9.htile.pipe_aligned);
+		}
+
+		if (rtex->dcc_offset) {
+			u_log_printf(log, "  DCC: offset=%"PRIu64", size=%"PRIu64", "
+				"alignment=%u, pitch_max=%u, num_dcc_levels=%u\n",
+				rtex->dcc_offset, rtex->surface.dcc_size,
+				rtex->surface.dcc_alignment,
+				rtex->surface.u.gfx9.dcc_pitch_max,
+				rtex->surface.num_dcc_levels);
+		}
+
+		if (rtex->surface.u.gfx9.stencil_offset) {
+			u_log_printf(log, "  Stencil: offset=%"PRIu64", swmode=%u, epitch=%u\n",
+				rtex->surface.u.gfx9.stencil_offset,
+				rtex->surface.u.gfx9.stencil.swizzle_mode,
+				rtex->surface.u.gfx9.stencil.epitch);
+		}
+		return;
+	}
+
+	u_log_printf(log, "  Layout: size=%"PRIu64", alignment=%u, bankw=%u, "
+		"bankh=%u, nbanks=%u, mtilea=%u, tilesplit=%u, pipeconfig=%u, scanout=%u\n",
+		rtex->surface.surf_size, rtex->surface.surf_alignment, rtex->surface.u.legacy.bankw,
+		rtex->surface.u.legacy.bankh, rtex->surface.u.legacy.num_banks, rtex->surface.u.legacy.mtilea,
+		rtex->surface.u.legacy.tile_split, rtex->surface.u.legacy.pipe_config,
+		(rtex->surface.flags & RADEON_SURF_SCANOUT) != 0);
+
+	if (rtex->fmask.size)
+		u_log_printf(log, "  FMask: offset=%"PRIu64", size=%"PRIu64", alignment=%u, pitch_in_pixels=%u, "
+			"bankh=%u, slice_tile_max=%u, tile_mode_index=%u\n",
+			rtex->fmask.offset, rtex->fmask.size, rtex->fmask.alignment,
+			rtex->fmask.pitch_in_pixels, rtex->fmask.bank_height,
+			rtex->fmask.slice_tile_max, rtex->fmask.tile_mode_index);
+
+	if (rtex->cmask.size)
+		u_log_printf(log, "  CMask: offset=%"PRIu64", size=%"PRIu64", alignment=%u, "
+			"slice_tile_max=%u\n",
+			rtex->cmask.offset, rtex->cmask.size, rtex->cmask.alignment,
+			rtex->cmask.slice_tile_max);
+
+	if (rtex->htile_offset)
+		u_log_printf(log, "  HTile: offset=%"PRIu64", size=%"PRIu64", "
+			"alignment=%u, TC_compatible = %u\n",
+			rtex->htile_offset, rtex->surface.htile_size,
+			rtex->surface.htile_alignment,
+			rtex->tc_compatible_htile);
+
+	if (rtex->dcc_offset) {
+		u_log_printf(log, "  DCC: offset=%"PRIu64", size=%"PRIu64", alignment=%u\n",
+			rtex->dcc_offset, rtex->surface.dcc_size,
+			rtex->surface.dcc_alignment);
+		for (i = 0; i <= rtex->resource.b.b.last_level; i++)
+			u_log_printf(log, "  DCCLevel[%i]: enabled=%u, offset=%"PRIu64", "
+				"fast_clear_size=%"PRIu64"\n",
+				i, i < rtex->surface.num_dcc_levels,
+				rtex->surface.u.legacy.level[i].dcc_offset,
+				rtex->surface.u.legacy.level[i].dcc_fast_clear_size);
+	}
+
+	for (i = 0; i <= rtex->resource.b.b.last_level; i++)
+		u_log_printf(log, "  Level[%i]: offset=%"PRIu64", slice_size=%"PRIu64", "
+			"npix_x=%u, npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
+			"mode=%u, tiling_index = %u\n",
+			i, rtex->surface.u.legacy.level[i].offset,
+			rtex->surface.u.legacy.level[i].slice_size,
+			u_minify(rtex->resource.b.b.width0, i),
+			u_minify(rtex->resource.b.b.height0, i),
+			u_minify(rtex->resource.b.b.depth0, i),
+			rtex->surface.u.legacy.level[i].nblk_x,
+			rtex->surface.u.legacy.level[i].nblk_y,
+			rtex->surface.u.legacy.level[i].mode,
+			rtex->surface.u.legacy.tiling_index[i]);
+
+	if (rtex->surface.has_stencil) {
+		u_log_printf(log, "  StencilLayout: tilesplit=%u\n",
+			rtex->surface.u.legacy.stencil_tile_split);
+		for (i = 0; i <= rtex->resource.b.b.last_level; i++) {
+			u_log_printf(log, "  StencilLevel[%i]: offset=%"PRIu64", "
+				"slice_size=%"PRIu64", npix_x=%u, "
+				"npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
+				"mode=%u, tiling_index = %u\n",
+				i, rtex->surface.u.legacy.stencil_level[i].offset,
+				rtex->surface.u.legacy.stencil_level[i].slice_size,
+				u_minify(rtex->resource.b.b.width0, i),
+				u_minify(rtex->resource.b.b.height0, i),
+				u_minify(rtex->resource.b.b.depth0, i),
+				rtex->surface.u.legacy.stencil_level[i].nblk_x,
+				rtex->surface.u.legacy.stencil_level[i].nblk_y,
+				rtex->surface.u.legacy.stencil_level[i].mode,
+				rtex->surface.u.legacy.stencil_tiling_index[i]);
+		}
+	}
+}
+
+/* Common processing for r600_texture_create and r600_texture_from_handle */
+static struct r600_texture *
+r600_texture_create_object(struct pipe_screen *screen,
+			   const struct pipe_resource *base,
+			   struct pb_buffer *buf,
+			   struct radeon_surf *surface)
+{
+	struct r600_texture *rtex;
+	struct r600_resource *resource;
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+
+	rtex = CALLOC_STRUCT(r600_texture);
+	if (!rtex)
+		return NULL;
+
+	resource = &rtex->resource;
+	resource->b.b = *base;
+	resource->b.b.next = NULL;
+	resource->b.vtbl = &r600_texture_vtbl;
+	pipe_reference_init(&resource->b.b.reference, 1);
+	resource->b.b.screen = screen;
+
+	/* don't include stencil-only formats which we don't support for rendering */
+	rtex->is_depth = util_format_has_depth(util_format_description(rtex->resource.b.b.format));
+
+	rtex->surface = *surface;
+	rtex->size = rtex->surface.surf_size;
+
+	rtex->tc_compatible_htile = rtex->surface.htile_size != 0 &&
+				    (rtex->surface.flags &
+				     RADEON_SURF_TC_COMPATIBLE_HTILE);
+
+	/* TC-compatible HTILE:
+	 * - VI only supports Z32_FLOAT.
+	 * - GFX9 only supports Z32_FLOAT and Z16_UNORM. */
+	if (rtex->tc_compatible_htile) {
+		if (rscreen->chip_class >= GFX9 &&
+		    base->format == PIPE_FORMAT_Z16_UNORM)
+			rtex->db_render_format = base->format;
+		else
+			rtex->db_render_format = PIPE_FORMAT_Z32_FLOAT;
+	} else {
+		rtex->db_render_format = base->format;
+	}
+
+	/* Tiled depth textures utilize the non-displayable tile order.
+	 * This must be done after r600_setup_surface.
+	 * Applies to R600-Cayman. */
+	rtex->non_disp_tiling = rtex->is_depth && rtex->surface.u.legacy.level[0].mode >= RADEON_SURF_MODE_1D;
+	/* Applies to GCN. */
+	rtex->last_msaa_resolve_target_micro_mode = rtex->surface.micro_tile_mode;
+
+	/* Disable separate DCC at the beginning. DRI2 doesn't reuse buffers
+	 * between frames, so the only thing that can enable separate DCC
+	 * with DRI2 is multiple slow clears within a frame.
+	 */
+	rtex->ps_draw_ratio = 0;
+
+	if (rtex->is_depth) {
+		if (base->flags & (R600_RESOURCE_FLAG_TRANSFER |
+				   R600_RESOURCE_FLAG_FLUSHED_DEPTH) ||
+		    rscreen->chip_class >= EVERGREEN) {
+			if (rscreen->chip_class >= GFX9) {
+				rtex->can_sample_z = true;
+				rtex->can_sample_s = true;
+			} else {
+				rtex->can_sample_z = !rtex->surface.u.legacy.depth_adjusted;
+				rtex->can_sample_s = !rtex->surface.u.legacy.stencil_adjusted;
+			}
+		} else {
+			if (rtex->resource.b.b.nr_samples <= 1 &&
+			    (rtex->resource.b.b.format == PIPE_FORMAT_Z16_UNORM ||
+			     rtex->resource.b.b.format == PIPE_FORMAT_Z32_FLOAT))
+				rtex->can_sample_z = true;
+		}
+
+		if (!(base->flags & (R600_RESOURCE_FLAG_TRANSFER |
+				     R600_RESOURCE_FLAG_FLUSHED_DEPTH))) {
+			rtex->db_compatible = true;
+
+			if (!(rscreen->debug_flags & DBG_NO_HYPERZ))
+				r600_texture_allocate_htile(rscreen, rtex);
+		}
+	} else {
+		if (base->nr_samples > 1) {
+			if (!buf) {
+				r600_texture_allocate_fmask(rscreen, rtex);
+				r600_texture_allocate_cmask(rscreen, rtex);
+				rtex->cmask_buffer = &rtex->resource;
+			}
+			if (!rtex->fmask.size || !rtex->cmask.size) {
+				FREE(rtex);
+				return NULL;
+			}
+		}
+
+		/* Shared textures must always set up DCC here.
+		 * If it's not present, it will be disabled by
+		 * apply_opaque_metadata later.
+		 */
+		if (rtex->surface.dcc_size &&
+		    (buf || !(rscreen->debug_flags & DBG_NO_DCC)) &&
+		    !(rtex->surface.flags & RADEON_SURF_SCANOUT)) {
+			/* Reserve space for the DCC buffer. */
+			rtex->dcc_offset = align64(rtex->size, rtex->surface.dcc_alignment);
+			rtex->size = rtex->dcc_offset + rtex->surface.dcc_size;
+		}
+	}
+
+	/* Now create the backing buffer. */
+	if (!buf) {
+		r600_init_resource_fields(rscreen, resource, rtex->size,
+					  rtex->surface.surf_alignment);
+
+		/* Displayable surfaces are not suballocated. */
+		if (resource->b.b.bind & PIPE_BIND_SCANOUT)
+			resource->flags |= RADEON_FLAG_NO_SUBALLOC;
+
+		if (!r600_alloc_resource(rscreen, resource)) {
+			FREE(rtex);
+			return NULL;
+		}
+	} else {
+		resource->buf = buf;
+		resource->gpu_address = rscreen->ws->buffer_get_virtual_address(resource->buf);
+		resource->bo_size = buf->size;
+		resource->bo_alignment = buf->alignment;
+		resource->domains = rscreen->ws->buffer_get_initial_domain(resource->buf);
+		if (resource->domains & RADEON_DOMAIN_VRAM)
+			resource->vram_usage = buf->size;
+		else if (resource->domains & RADEON_DOMAIN_GTT)
+			resource->gart_usage = buf->size;
+	}
+
+	if (rtex->cmask.size) {
+		/* Initialize the cmask to 0xCC (= compressed state). */
+		r600_screen_clear_buffer(rscreen, &rtex->cmask_buffer->b.b,
+					 rtex->cmask.offset, rtex->cmask.size,
+					 0xCCCCCCCC);
+	}
+	if (rtex->htile_offset) {
+		uint32_t clear_value = 0;
+
+		if (rscreen->chip_class >= GFX9 || rtex->tc_compatible_htile)
+			clear_value = 0x0000030F;
+
+		r600_screen_clear_buffer(rscreen, &rtex->resource.b.b,
+					 rtex->htile_offset,
+					 rtex->surface.htile_size,
+					 clear_value);
+	}
+
+	/* Initialize DCC only if the texture is not being imported. */
+	if (!buf && rtex->dcc_offset) {
+		r600_screen_clear_buffer(rscreen, &rtex->resource.b.b,
+					 rtex->dcc_offset,
+					 rtex->surface.dcc_size,
+					 0xFFFFFFFF);
+	}
+
+	/* Initialize the CMASK base register value. */
+	rtex->cmask.base_address_reg =
+		(rtex->resource.gpu_address + rtex->cmask.offset) >> 8;
+
+	if (rscreen->debug_flags & DBG_VM) {
+		fprintf(stderr, "VM start=0x%"PRIX64"  end=0x%"PRIX64" | Texture %ix%ix%i, %i levels, %i samples, %s\n",
+			rtex->resource.gpu_address,
+			rtex->resource.gpu_address + rtex->resource.buf->size,
+			base->width0, base->height0, util_max_layer(base, 0)+1, base->last_level+1,
+			base->nr_samples ? base->nr_samples : 1, util_format_short_name(base->format));
+	}
+
+	if (rscreen->debug_flags & DBG_TEX) {
+		puts("Texture:");
+		struct u_log_context log;
+		u_log_context_init(&log);
+		r600_print_texture_info(rscreen, rtex, &log);
+		u_log_new_page_print(&log, stdout);
+		fflush(stdout);
+		u_log_context_destroy(&log);
+	}
+
+	return rtex;
+}
+
+static enum radeon_surf_mode
+r600_choose_tiling(struct r600_common_screen *rscreen,
+		   const struct pipe_resource *templ)
+{
+	const struct util_format_description *desc = util_format_description(templ->format);
+	bool force_tiling = templ->flags & R600_RESOURCE_FLAG_FORCE_TILING;
+	bool is_depth_stencil = util_format_is_depth_or_stencil(templ->format) &&
+				!(templ->flags & R600_RESOURCE_FLAG_FLUSHED_DEPTH);
+
+	/* MSAA resources must be 2D tiled. */
+	if (templ->nr_samples > 1)
+		return RADEON_SURF_MODE_2D;
+
+	/* Transfer resources should be linear. */
+	if (templ->flags & R600_RESOURCE_FLAG_TRANSFER)
+		return RADEON_SURF_MODE_LINEAR_ALIGNED;
+
+	/* Avoid Z/S decompress blits by forcing TC-compatible HTILE on VI,
+	 * which requires 2D tiling.
+	 */
+	if (rscreen->chip_class == VI &&
+	    is_depth_stencil &&
+	    (templ->flags & PIPE_RESOURCE_FLAG_TEXTURING_MORE_LIKELY))
+		return RADEON_SURF_MODE_2D;
+
+	/* r600g: force tiling on TEXTURE_2D and TEXTURE_3D compute resources. */
+	if (rscreen->chip_class >= R600 && rscreen->chip_class <= CAYMAN &&
+	    (templ->bind & PIPE_BIND_COMPUTE_RESOURCE) &&
+	    (templ->target == PIPE_TEXTURE_2D ||
+	     templ->target == PIPE_TEXTURE_3D))
+		force_tiling = true;
+
+	/* Handle common candidates for the linear mode.
+	 * Compressed textures and DB surfaces must always be tiled.
+	 */
+	if (!force_tiling &&
+	    !is_depth_stencil &&
+	    !util_format_is_compressed(templ->format)) {
+		if (rscreen->debug_flags & DBG_NO_TILING)
+			return RADEON_SURF_MODE_LINEAR_ALIGNED;
+
+		/* Tiling doesn't work with the 422 (SUBSAMPLED) formats on R600+. */
+		if (desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED)
+			return RADEON_SURF_MODE_LINEAR_ALIGNED;
+
+		/* Cursors are linear on SI.
+		 * (XXX double-check, maybe also use RADEON_SURF_SCANOUT) */
+		if (rscreen->chip_class >= SI &&
+		    (templ->bind & PIPE_BIND_CURSOR))
+			return RADEON_SURF_MODE_LINEAR_ALIGNED;
+
+		if (templ->bind & PIPE_BIND_LINEAR)
+			return RADEON_SURF_MODE_LINEAR_ALIGNED;
+
+		/* Textures with a very small height are recommended to be linear. */
+		if (templ->target == PIPE_TEXTURE_1D ||
+		    templ->target == PIPE_TEXTURE_1D_ARRAY ||
+		    /* Only very thin and long 2D textures should benefit from
+		     * linear_aligned. */
+		    (templ->width0 > 8 && templ->height0 <= 2))
+			return RADEON_SURF_MODE_LINEAR_ALIGNED;
+
+		/* Textures likely to be mapped often. */
+		if (templ->usage == PIPE_USAGE_STAGING ||
+		    templ->usage == PIPE_USAGE_STREAM)
+			return RADEON_SURF_MODE_LINEAR_ALIGNED;
+	}
+
+	/* Make small textures 1D tiled. */
+	if (templ->width0 <= 16 || templ->height0 <= 16 ||
+	    (rscreen->debug_flags & DBG_NO_2D_TILING))
+		return RADEON_SURF_MODE_1D;
+
+	/* The allocator will switch to 1D if needed. */
+	return RADEON_SURF_MODE_2D;
+}
+
+struct pipe_resource *r600_texture_create(struct pipe_screen *screen,
+					  const struct pipe_resource *templ)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+	struct radeon_surf surface = {0};
+	bool is_flushed_depth = templ->flags & R600_RESOURCE_FLAG_FLUSHED_DEPTH;
+	bool tc_compatible_htile =
+		rscreen->chip_class >= VI &&
+		(templ->flags & PIPE_RESOURCE_FLAG_TEXTURING_MORE_LIKELY) &&
+		!(rscreen->debug_flags & DBG_NO_HYPERZ) &&
+		!is_flushed_depth &&
+		templ->nr_samples <= 1 && /* TC-compat HTILE is less efficient with MSAA */
+		util_format_is_depth_or_stencil(templ->format);
+
+	int r;
+
+	r = r600_init_surface(rscreen, &surface, templ,
+			      r600_choose_tiling(rscreen, templ), 0, 0,
+			      false, false, is_flushed_depth,
+			      tc_compatible_htile);
+	if (r) {
+		return NULL;
+	}
+
+	return (struct pipe_resource *)
+	       r600_texture_create_object(screen, templ, NULL, &surface);
+}
+
+static struct pipe_resource *r600_texture_from_handle(struct pipe_screen *screen,
+						      const struct pipe_resource *templ,
+						      struct winsys_handle *whandle,
+                                                      unsigned usage)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+	struct pb_buffer *buf = NULL;
+	unsigned stride = 0, offset = 0;
+	enum radeon_surf_mode array_mode;
+	struct radeon_surf surface = {};
+	int r;
+	struct radeon_bo_metadata metadata = {};
+	struct r600_texture *rtex;
+	bool is_scanout;
+
+	/* Support only 2D textures without mipmaps */
+	if ((templ->target != PIPE_TEXTURE_2D && templ->target != PIPE_TEXTURE_RECT) ||
+	      templ->depth0 != 1 || templ->last_level != 0)
+		return NULL;
+
+	buf = rscreen->ws->buffer_from_handle(rscreen->ws, whandle, &stride, &offset);
+	if (!buf)
+		return NULL;
+
+	rscreen->ws->buffer_get_metadata(buf, &metadata);
+	r600_surface_import_metadata(rscreen, &surface, &metadata,
+				     &array_mode, &is_scanout);
+
+	r = r600_init_surface(rscreen, &surface, templ, array_mode, stride,
+			      offset, true, is_scanout, false, false);
+	if (r) {
+		return NULL;
+	}
+
+	rtex = r600_texture_create_object(screen, templ, buf, &surface);
+	if (!rtex)
+		return NULL;
+
+	rtex->resource.b.is_shared = true;
+	rtex->resource.external_usage = usage;
+
+	if (rscreen->apply_opaque_metadata)
+		rscreen->apply_opaque_metadata(rscreen, rtex, &metadata);
+
+	assert(rtex->surface.tile_swizzle == 0);
+	return &rtex->resource.b.b;
+}
+
+bool r600_init_flushed_depth_texture(struct pipe_context *ctx,
+				     struct pipe_resource *texture,
+				     struct r600_texture **staging)
+{
+	struct r600_texture *rtex = (struct r600_texture*)texture;
+	struct pipe_resource resource;
+	struct r600_texture **flushed_depth_texture = staging ?
+			staging : &rtex->flushed_depth_texture;
+	enum pipe_format pipe_format = texture->format;
+
+	if (!staging) {
+		if (rtex->flushed_depth_texture)
+			return true; /* it's ready */
+
+		if (!rtex->can_sample_z && rtex->can_sample_s) {
+			switch (pipe_format) {
+			case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
+				/* Save memory by not allocating the S plane. */
+				pipe_format = PIPE_FORMAT_Z32_FLOAT;
+				break;
+			case PIPE_FORMAT_Z24_UNORM_S8_UINT:
+			case PIPE_FORMAT_S8_UINT_Z24_UNORM:
+				/* Save memory bandwidth by not copying the
+				 * stencil part during flush.
+				 *
+				 * This potentially increases memory bandwidth
+				 * if an application uses both Z and S texturing
+				 * simultaneously (a flushed Z24S8 texture
+				 * would be stored compactly), but how often
+				 * does that really happen?
+				 */
+				pipe_format = PIPE_FORMAT_Z24X8_UNORM;
+				break;
+			default:;
+			}
+		} else if (!rtex->can_sample_s && rtex->can_sample_z) {
+			assert(util_format_has_stencil(util_format_description(pipe_format)));
+
+			/* DB->CB copies to an 8bpp surface don't work. */
+			pipe_format = PIPE_FORMAT_X24S8_UINT;
+		}
+	}
+
+	memset(&resource, 0, sizeof(resource));
+	resource.target = texture->target;
+	resource.format = pipe_format;
+	resource.width0 = texture->width0;
+	resource.height0 = texture->height0;
+	resource.depth0 = texture->depth0;
+	resource.array_size = texture->array_size;
+	resource.last_level = texture->last_level;
+	resource.nr_samples = texture->nr_samples;
+	resource.usage = staging ? PIPE_USAGE_STAGING : PIPE_USAGE_DEFAULT;
+	resource.bind = texture->bind & ~PIPE_BIND_DEPTH_STENCIL;
+	resource.flags = texture->flags | R600_RESOURCE_FLAG_FLUSHED_DEPTH;
+
+	if (staging)
+		resource.flags |= R600_RESOURCE_FLAG_TRANSFER;
+
+	*flushed_depth_texture = (struct r600_texture *)ctx->screen->resource_create(ctx->screen, &resource);
+	if (*flushed_depth_texture == NULL) {
+		R600_ERR("failed to create temporary texture to hold flushed depth\n");
+		return false;
+	}
+
+	(*flushed_depth_texture)->non_disp_tiling = false;
+	return true;
+}
+
+/**
+ * Initialize the pipe_resource descriptor to be of the same size as the box,
+ * which is supposed to hold a subregion of the texture "orig" at the given
+ * mipmap level.
+ */
+static void r600_init_temp_resource_from_box(struct pipe_resource *res,
+					     struct pipe_resource *orig,
+					     const struct pipe_box *box,
+					     unsigned level, unsigned flags)
+{
+	memset(res, 0, sizeof(*res));
+	res->format = orig->format;
+	res->width0 = box->width;
+	res->height0 = box->height;
+	res->depth0 = 1;
+	res->array_size = 1;
+	res->usage = flags & R600_RESOURCE_FLAG_TRANSFER ? PIPE_USAGE_STAGING : PIPE_USAGE_DEFAULT;
+	res->flags = flags;
+
+	/* We must set the correct texture target and dimensions for a 3D box. */
+	if (box->depth > 1 && util_max_layer(orig, level) > 0) {
+		res->target = PIPE_TEXTURE_2D_ARRAY;
+		res->array_size = box->depth;
+	} else {
+		res->target = PIPE_TEXTURE_2D;
+	}
+}
+
+static bool r600_can_invalidate_texture(struct r600_common_screen *rscreen,
+					struct r600_texture *rtex,
+					unsigned transfer_usage,
+					const struct pipe_box *box)
+{
+	/* r600g doesn't react to dirty_tex_descriptor_counter */
+	return rscreen->chip_class >= SI &&
+		!rtex->resource.b.is_shared &&
+		!(transfer_usage & PIPE_TRANSFER_READ) &&
+		rtex->resource.b.b.last_level == 0 &&
+		util_texrange_covers_whole_level(&rtex->resource.b.b, 0,
+						 box->x, box->y, box->z,
+						 box->width, box->height,
+						 box->depth);
+}
+
+static void r600_texture_invalidate_storage(struct r600_common_context *rctx,
+					    struct r600_texture *rtex)
+{
+	struct r600_common_screen *rscreen = rctx->screen;
+
+	/* There is no point in discarding depth and tiled buffers. */
+	assert(!rtex->is_depth);
+	assert(rtex->surface.is_linear);
+
+	/* Reallocate the buffer in the same pipe_resource. */
+	r600_alloc_resource(rscreen, &rtex->resource);
+
+	/* Initialize the CMASK base address (needed even without CMASK). */
+	rtex->cmask.base_address_reg =
+		(rtex->resource.gpu_address + rtex->cmask.offset) >> 8;
+
+	p_atomic_inc(&rscreen->dirty_tex_counter);
+
+	rctx->num_alloc_tex_transfer_bytes += rtex->size;
+}
+
+static void *r600_texture_transfer_map(struct pipe_context *ctx,
+				       struct pipe_resource *texture,
+				       unsigned level,
+				       unsigned usage,
+				       const struct pipe_box *box,
+				       struct pipe_transfer **ptransfer)
+{
+	struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+	struct r600_texture *rtex = (struct r600_texture*)texture;
+	struct r600_transfer *trans;
+	struct r600_resource *buf;
+	unsigned offset = 0;
+	char *map;
+	bool use_staging_texture = false;
+
+	assert(!(texture->flags & R600_RESOURCE_FLAG_TRANSFER));
+	assert(box->width && box->height && box->depth);
+
+	/* Depth textures use staging unconditionally. */
+	if (!rtex->is_depth) {
+		/* Degrade the tile mode if we get too many transfers on APUs.
+		 * On dGPUs, the staging texture is always faster.
+		 * Only count uploads that are at least 4x4 pixels large.
+		 */
+		if (!rctx->screen->info.has_dedicated_vram &&
+		    level == 0 &&
+		    box->width >= 4 && box->height >= 4 &&
+		    p_atomic_inc_return(&rtex->num_level0_transfers) == 10) {
+			bool can_invalidate =
+				r600_can_invalidate_texture(rctx->screen, rtex,
+							    usage, box);
+
+			r600_reallocate_texture_inplace(rctx, rtex,
+							PIPE_BIND_LINEAR,
+							can_invalidate);
+		}
+
+		/* Tiled textures need to be converted into a linear texture for CPU
+		 * access. The staging texture is always linear and is placed in GART.
+		 *
+		 * Reading from VRAM or GTT WC is slow, always use the staging
+		 * texture in this case.
+		 *
+		 * Use the staging texture for uploads if the underlying BO
+		 * is busy.
+		 */
+		if (!rtex->surface.is_linear)
+			use_staging_texture = true;
+		else if (usage & PIPE_TRANSFER_READ)
+			use_staging_texture =
+				rtex->resource.domains & RADEON_DOMAIN_VRAM ||
+				rtex->resource.flags & RADEON_FLAG_GTT_WC;
+		/* Write & linear only: */
+		else if (r600_rings_is_buffer_referenced(rctx, rtex->resource.buf,
+							 RADEON_USAGE_READWRITE) ||
+			 !rctx->ws->buffer_wait(rtex->resource.buf, 0,
+						RADEON_USAGE_READWRITE)) {
+			/* It's busy. */
+			if (r600_can_invalidate_texture(rctx->screen, rtex,
+							usage, box))
+				r600_texture_invalidate_storage(rctx, rtex);
+			else
+				use_staging_texture = true;
+		}
+	}
+
+	trans = CALLOC_STRUCT(r600_transfer);
+	if (!trans)
+		return NULL;
+	pipe_resource_reference(&trans->b.b.resource, texture);
+	trans->b.b.level = level;
+	trans->b.b.usage = usage;
+	trans->b.b.box = *box;
+
+	if (rtex->is_depth) {
+		struct r600_texture *staging_depth;
+
+		if (rtex->resource.b.b.nr_samples > 1) {
+			/* MSAA depth buffers need to be converted to single sample buffers.
+			 *
+			 * Mapping MSAA depth buffers can occur if ReadPixels is called
+			 * with a multisample GLX visual.
+			 *
+			 * First downsample the depth buffer to a temporary texture,
+			 * then decompress the temporary one to staging.
+			 *
+			 * Only the region being mapped is transfered.
+			 */
+			struct pipe_resource resource;
+
+			r600_init_temp_resource_from_box(&resource, texture, box, level, 0);
+
+			if (!r600_init_flushed_depth_texture(ctx, &resource, &staging_depth)) {
+				R600_ERR("failed to create temporary texture to hold untiled copy\n");
+				FREE(trans);
+				return NULL;
+			}
+
+			if (usage & PIPE_TRANSFER_READ) {
+				struct pipe_resource *temp = ctx->screen->resource_create(ctx->screen, &resource);
+				if (!temp) {
+					R600_ERR("failed to create a temporary depth texture\n");
+					FREE(trans);
+					return NULL;
+				}
+
+				r600_copy_region_with_blit(ctx, temp, 0, 0, 0, 0, texture, level, box);
+				rctx->blit_decompress_depth(ctx, (struct r600_texture*)temp, staging_depth,
+							    0, 0, 0, box->depth, 0, 0);
+				pipe_resource_reference(&temp, NULL);
+			}
+
+			/* Just get the strides. */
+			r600_texture_get_offset(rctx->screen, staging_depth, level, NULL,
+						&trans->b.b.stride,
+						&trans->b.b.layer_stride);
+		} else {
+			/* XXX: only readback the rectangle which is being mapped? */
+			/* XXX: when discard is true, no need to read back from depth texture */
+			if (!r600_init_flushed_depth_texture(ctx, texture, &staging_depth)) {
+				R600_ERR("failed to create temporary texture to hold untiled copy\n");
+				FREE(trans);
+				return NULL;
+			}
+
+			rctx->blit_decompress_depth(ctx, rtex, staging_depth,
+						    level, level,
+						    box->z, box->z + box->depth - 1,
+						    0, 0);
+
+			offset = r600_texture_get_offset(rctx->screen, staging_depth,
+							 level, box,
+							 &trans->b.b.stride,
+							 &trans->b.b.layer_stride);
+		}
+
+		trans->staging = (struct r600_resource*)staging_depth;
+		buf = trans->staging;
+	} else if (use_staging_texture) {
+		struct pipe_resource resource;
+		struct r600_texture *staging;
+
+		r600_init_temp_resource_from_box(&resource, texture, box, level,
+						 R600_RESOURCE_FLAG_TRANSFER);
+		resource.usage = (usage & PIPE_TRANSFER_READ) ?
+			PIPE_USAGE_STAGING : PIPE_USAGE_STREAM;
+
+		/* Create the temporary texture. */
+		staging = (struct r600_texture*)ctx->screen->resource_create(ctx->screen, &resource);
+		if (!staging) {
+			R600_ERR("failed to create temporary texture to hold untiled copy\n");
+			FREE(trans);
+			return NULL;
+		}
+		trans->staging = &staging->resource;
+
+		/* Just get the strides. */
+		r600_texture_get_offset(rctx->screen, staging, 0, NULL,
+					&trans->b.b.stride,
+					&trans->b.b.layer_stride);
+
+		if (usage & PIPE_TRANSFER_READ)
+			r600_copy_to_staging_texture(ctx, trans);
+		else
+			usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
+
+		buf = trans->staging;
+	} else {
+		/* the resource is mapped directly */
+		offset = r600_texture_get_offset(rctx->screen, rtex, level, box,
+						 &trans->b.b.stride,
+						 &trans->b.b.layer_stride);
+		buf = &rtex->resource;
+	}
+
+	if (!(map = r600_buffer_map_sync_with_rings(rctx, buf, usage))) {
+		r600_resource_reference(&trans->staging, NULL);
+		FREE(trans);
+		return NULL;
+	}
+
+	*ptransfer = &trans->b.b;
+	return map + offset;
+}
+
+static void r600_texture_transfer_unmap(struct pipe_context *ctx,
+					struct pipe_transfer* transfer)
+{
+	struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+	struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
+	struct pipe_resource *texture = transfer->resource;
+	struct r600_texture *rtex = (struct r600_texture*)texture;
+
+	if ((transfer->usage & PIPE_TRANSFER_WRITE) && rtransfer->staging) {
+		if (rtex->is_depth && rtex->resource.b.b.nr_samples <= 1) {
+			ctx->resource_copy_region(ctx, texture, transfer->level,
+						  transfer->box.x, transfer->box.y, transfer->box.z,
+						  &rtransfer->staging->b.b, transfer->level,
+						  &transfer->box);
+		} else {
+			r600_copy_from_staging_texture(ctx, rtransfer);
+		}
+	}
+
+	if (rtransfer->staging) {
+		rctx->num_alloc_tex_transfer_bytes += rtransfer->staging->buf->size;
+		r600_resource_reference(&rtransfer->staging, NULL);
+	}
+
+	/* Heuristic for {upload, draw, upload, draw, ..}:
+	 *
+	 * Flush the gfx IB if we've allocated too much texture storage.
+	 *
+	 * The idea is that we don't want to build IBs that use too much
+	 * memory and put pressure on the kernel memory manager and we also
+	 * want to make temporary and invalidated buffers go idle ASAP to
+	 * decrease the total memory usage or make them reusable. The memory
+	 * usage will be slightly higher than given here because of the buffer
+	 * cache in the winsys.
+	 *
+	 * The result is that the kernel memory manager is never a bottleneck.
+	 */
+	if (rctx->num_alloc_tex_transfer_bytes > rctx->screen->info.gart_size / 4) {
+		rctx->gfx.flush(rctx, RADEON_FLUSH_ASYNC, NULL);
+		rctx->num_alloc_tex_transfer_bytes = 0;
+	}
+
+	pipe_resource_reference(&transfer->resource, NULL);
+	FREE(transfer);
+}
+
+static const struct u_resource_vtbl r600_texture_vtbl =
+{
+	NULL,				/* get_handle */
+	r600_texture_destroy,		/* resource_destroy */
+	r600_texture_transfer_map,	/* transfer_map */
+	u_default_transfer_flush_region, /* transfer_flush_region */
+	r600_texture_transfer_unmap,	/* transfer_unmap */
+};
+
+struct pipe_surface *r600_create_surface_custom(struct pipe_context *pipe,
+						struct pipe_resource *texture,
+						const struct pipe_surface *templ,
+						unsigned width0, unsigned height0,
+						unsigned width, unsigned height)
+{
+	struct r600_surface *surface = CALLOC_STRUCT(r600_surface);
+
+	if (!surface)
+		return NULL;
+
+	assert(templ->u.tex.first_layer <= util_max_layer(texture, templ->u.tex.level));
+	assert(templ->u.tex.last_layer <= util_max_layer(texture, templ->u.tex.level));
+
+	pipe_reference_init(&surface->base.reference, 1);
+	pipe_resource_reference(&surface->base.texture, texture);
+	surface->base.context = pipe;
+	surface->base.format = templ->format;
+	surface->base.width = width;
+	surface->base.height = height;
+	surface->base.u = templ->u;
+
+	surface->width0 = width0;
+	surface->height0 = height0;
+
+	return &surface->base;
+}
+
+static struct pipe_surface *r600_create_surface(struct pipe_context *pipe,
+						struct pipe_resource *tex,
+						const struct pipe_surface *templ)
+{
+	unsigned level = templ->u.tex.level;
+	unsigned width = u_minify(tex->width0, level);
+	unsigned height = u_minify(tex->height0, level);
+	unsigned width0 = tex->width0;
+	unsigned height0 = tex->height0;
+
+	if (tex->target != PIPE_BUFFER && templ->format != tex->format) {
+		const struct util_format_description *tex_desc
+			= util_format_description(tex->format);
+		const struct util_format_description *templ_desc
+			= util_format_description(templ->format);
+
+		assert(tex_desc->block.bits == templ_desc->block.bits);
+
+		/* Adjust size of surface if and only if the block width or
+		 * height is changed. */
+		if (tex_desc->block.width != templ_desc->block.width ||
+		    tex_desc->block.height != templ_desc->block.height) {
+			unsigned nblks_x = util_format_get_nblocksx(tex->format, width);
+			unsigned nblks_y = util_format_get_nblocksy(tex->format, height);
+
+			width = nblks_x * templ_desc->block.width;
+			height = nblks_y * templ_desc->block.height;
+
+			width0 = util_format_get_nblocksx(tex->format, width0);
+			height0 = util_format_get_nblocksy(tex->format, height0);
+		}
+	}
+
+	return r600_create_surface_custom(pipe, tex, templ,
+					  width0, height0,
+					  width, height);
+}
+
+static void r600_surface_destroy(struct pipe_context *pipe,
+				 struct pipe_surface *surface)
+{
+	struct r600_surface *surf = (struct r600_surface*)surface;
+	r600_resource_reference(&surf->cb_buffer_fmask, NULL);
+	r600_resource_reference(&surf->cb_buffer_cmask, NULL);
+	pipe_resource_reference(&surface->texture, NULL);
+	FREE(surface);
+}
+
+static void r600_clear_texture(struct pipe_context *pipe,
+			       struct pipe_resource *tex,
+			       unsigned level,
+			       const struct pipe_box *box,
+			       const void *data)
+{
+	struct pipe_screen *screen = pipe->screen;
+	struct r600_texture *rtex = (struct r600_texture*)tex;
+	struct pipe_surface tmpl = {{0}};
+	struct pipe_surface *sf;
+	const struct util_format_description *desc =
+		util_format_description(tex->format);
+
+	tmpl.format = tex->format;
+	tmpl.u.tex.first_layer = box->z;
+	tmpl.u.tex.last_layer = box->z + box->depth - 1;
+	tmpl.u.tex.level = level;
+	sf = pipe->create_surface(pipe, tex, &tmpl);
+	if (!sf)
+		return;
+
+	if (rtex->is_depth) {
+		unsigned clear;
+		float depth;
+		uint8_t stencil = 0;
+
+		/* Depth is always present. */
+		clear = PIPE_CLEAR_DEPTH;
+		desc->unpack_z_float(&depth, 0, data, 0, 1, 1);
+
+		if (rtex->surface.has_stencil) {
+			clear |= PIPE_CLEAR_STENCIL;
+			desc->unpack_s_8uint(&stencil, 0, data, 0, 1, 1);
+		}
+
+		pipe->clear_depth_stencil(pipe, sf, clear, depth, stencil,
+					  box->x, box->y,
+					  box->width, box->height, false);
+	} else {
+		union pipe_color_union color;
+
+		/* pipe_color_union requires the full vec4 representation. */
+		if (util_format_is_pure_uint(tex->format))
+			desc->unpack_rgba_uint(color.ui, 0, data, 0, 1, 1);
+		else if (util_format_is_pure_sint(tex->format))
+			desc->unpack_rgba_sint(color.i, 0, data, 0, 1, 1);
+		else
+			desc->unpack_rgba_float(color.f, 0, data, 0, 1, 1);
+
+		if (screen->is_format_supported(screen, tex->format,
+						tex->target, 0,
+						PIPE_BIND_RENDER_TARGET)) {
+			pipe->clear_render_target(pipe, sf, &color,
+						  box->x, box->y,
+						  box->width, box->height, false);
+		} else {
+			/* Software fallback - just for R9G9B9E5_FLOAT */
+			util_clear_render_target(pipe, sf, &color,
+						 box->x, box->y,
+						 box->width, box->height);
+		}
+	}
+	pipe_surface_reference(&sf, NULL);
+}
+
+unsigned r600_translate_colorswap(enum pipe_format format, bool do_endian_swap)
+{
+	const struct util_format_description *desc = util_format_description(format);
+
+#define HAS_SWIZZLE(chan,swz) (desc->swizzle[chan] == PIPE_SWIZZLE_##swz)
+
+	if (format == PIPE_FORMAT_R11G11B10_FLOAT) /* isn't plain */
+		return V_0280A0_SWAP_STD;
+
+	if (desc->layout != UTIL_FORMAT_LAYOUT_PLAIN)
+		return ~0U;
+
+	switch (desc->nr_channels) {
+	case 1:
+		if (HAS_SWIZZLE(0,X))
+			return V_0280A0_SWAP_STD; /* X___ */
+		else if (HAS_SWIZZLE(3,X))
+			return V_0280A0_SWAP_ALT_REV; /* ___X */
+		break;
+	case 2:
+		if ((HAS_SWIZZLE(0,X) && HAS_SWIZZLE(1,Y)) ||
+		    (HAS_SWIZZLE(0,X) && HAS_SWIZZLE(1,NONE)) ||
+		    (HAS_SWIZZLE(0,NONE) && HAS_SWIZZLE(1,Y)))
+			return V_0280A0_SWAP_STD; /* XY__ */
+		else if ((HAS_SWIZZLE(0,Y) && HAS_SWIZZLE(1,X)) ||
+			 (HAS_SWIZZLE(0,Y) && HAS_SWIZZLE(1,NONE)) ||
+		         (HAS_SWIZZLE(0,NONE) && HAS_SWIZZLE(1,X)))
+			/* YX__ */
+			return (do_endian_swap ? V_0280A0_SWAP_STD : V_0280A0_SWAP_STD_REV);
+		else if (HAS_SWIZZLE(0,X) && HAS_SWIZZLE(3,Y))
+			return V_0280A0_SWAP_ALT; /* X__Y */
+		else if (HAS_SWIZZLE(0,Y) && HAS_SWIZZLE(3,X))
+			return V_0280A0_SWAP_ALT_REV; /* Y__X */
+		break;
+	case 3:
+		if (HAS_SWIZZLE(0,X))
+			return (do_endian_swap ? V_0280A0_SWAP_STD_REV : V_0280A0_SWAP_STD);
+		else if (HAS_SWIZZLE(0,Z))
+			return V_0280A0_SWAP_STD_REV; /* ZYX */
+		break;
+	case 4:
+		/* check the middle channels, the 1st and 4th channel can be NONE */
+		if (HAS_SWIZZLE(1,Y) && HAS_SWIZZLE(2,Z)) {
+			return V_0280A0_SWAP_STD; /* XYZW */
+		} else if (HAS_SWIZZLE(1,Z) && HAS_SWIZZLE(2,Y)) {
+			return V_0280A0_SWAP_STD_REV; /* WZYX */
+		} else if (HAS_SWIZZLE(1,Y) && HAS_SWIZZLE(2,X)) {
+			return V_0280A0_SWAP_ALT; /* ZYXW */
+		} else if (HAS_SWIZZLE(1,Z) && HAS_SWIZZLE(2,W)) {
+			/* YZWX */
+			if (desc->is_array)
+				return V_0280A0_SWAP_ALT_REV;
+			else
+				return (do_endian_swap ? V_0280A0_SWAP_ALT : V_0280A0_SWAP_ALT_REV);
+		}
+		break;
+	}
+	return ~0U;
+}
+
+/* FAST COLOR CLEAR */
+
+static void evergreen_set_clear_color(struct r600_texture *rtex,
+				      enum pipe_format surface_format,
+				      const union pipe_color_union *color)
+{
+	union util_color uc;
+
+	memset(&uc, 0, sizeof(uc));
+
+	if (rtex->surface.bpe == 16) {
+		/* DCC fast clear only:
+		 *   CLEAR_WORD0 = R = G = B
+		 *   CLEAR_WORD1 = A
+		 */
+		assert(color->ui[0] == color->ui[1] &&
+		       color->ui[0] == color->ui[2]);
+		uc.ui[0] = color->ui[0];
+		uc.ui[1] = color->ui[3];
+	} else if (util_format_is_pure_uint(surface_format)) {
+		util_format_write_4ui(surface_format, color->ui, 0, &uc, 0, 0, 0, 1, 1);
+	} else if (util_format_is_pure_sint(surface_format)) {
+		util_format_write_4i(surface_format, color->i, 0, &uc, 0, 0, 0, 1, 1);
+	} else {
+		util_pack_color(color->f, surface_format, &uc);
+	}
+
+	memcpy(rtex->color_clear_value, &uc, 2 * sizeof(uint32_t));
+}
+
+/* Set the same micro tile mode as the destination of the last MSAA resolve.
+ * This allows hitting the MSAA resolve fast path, which requires that both
+ * src and dst micro tile modes match.
+ */
+static void si_set_optimal_micro_tile_mode(struct r600_common_screen *rscreen,
+					   struct r600_texture *rtex)
+{
+	if (rtex->resource.b.is_shared ||
+	    rtex->resource.b.b.nr_samples <= 1 ||
+	    rtex->surface.micro_tile_mode == rtex->last_msaa_resolve_target_micro_mode)
+		return;
+
+	assert(rscreen->chip_class >= GFX9 ||
+	       rtex->surface.u.legacy.level[0].mode == RADEON_SURF_MODE_2D);
+	assert(rtex->resource.b.b.last_level == 0);
+
+	if (rscreen->chip_class >= GFX9) {
+		/* 4K or larger tiles only. 0 is linear. 1-3 are 256B tiles. */
+		assert(rtex->surface.u.gfx9.surf.swizzle_mode >= 4);
+
+		/* If you do swizzle_mode % 4, you'll get:
+		 *   0 = Depth
+		 *   1 = Standard,
+		 *   2 = Displayable
+		 *   3 = Rotated
+		 *
+		 * Depth-sample order isn't allowed:
+		 */
+		assert(rtex->surface.u.gfx9.surf.swizzle_mode % 4 != 0);
+
+		switch (rtex->last_msaa_resolve_target_micro_mode) {
+		case RADEON_MICRO_MODE_DISPLAY:
+			rtex->surface.u.gfx9.surf.swizzle_mode &= ~0x3;
+			rtex->surface.u.gfx9.surf.swizzle_mode += 2; /* D */
+			break;
+		case RADEON_MICRO_MODE_THIN:
+			rtex->surface.u.gfx9.surf.swizzle_mode &= ~0x3;
+			rtex->surface.u.gfx9.surf.swizzle_mode += 1; /* S */
+			break;
+		case RADEON_MICRO_MODE_ROTATED:
+			rtex->surface.u.gfx9.surf.swizzle_mode &= ~0x3;
+			rtex->surface.u.gfx9.surf.swizzle_mode += 3; /* R */
+			break;
+		default: /* depth */
+			assert(!"unexpected micro mode");
+			return;
+		}
+	} else if (rscreen->chip_class >= CIK) {
+		/* These magic numbers were copied from addrlib. It doesn't use
+		 * any definitions for them either. They are all 2D_TILED_THIN1
+		 * modes with different bpp and micro tile mode.
+		 */
+		switch (rtex->last_msaa_resolve_target_micro_mode) {
+		case RADEON_MICRO_MODE_DISPLAY:
+			rtex->surface.u.legacy.tiling_index[0] = 10;
+			break;
+		case RADEON_MICRO_MODE_THIN:
+			rtex->surface.u.legacy.tiling_index[0] = 14;
+			break;
+		case RADEON_MICRO_MODE_ROTATED:
+			rtex->surface.u.legacy.tiling_index[0] = 28;
+			break;
+		default: /* depth, thick */
+			assert(!"unexpected micro mode");
+			return;
+		}
+	} else { /* SI */
+		switch (rtex->last_msaa_resolve_target_micro_mode) {
+		case RADEON_MICRO_MODE_DISPLAY:
+			switch (rtex->surface.bpe) {
+			case 1:
+                            rtex->surface.u.legacy.tiling_index[0] = 10;
+                            break;
+			case 2:
+                            rtex->surface.u.legacy.tiling_index[0] = 11;
+                            break;
+			default: /* 4, 8 */
+                            rtex->surface.u.legacy.tiling_index[0] = 12;
+                            break;
+			}
+			break;
+		case RADEON_MICRO_MODE_THIN:
+			switch (rtex->surface.bpe) {
+			case 1:
+                                rtex->surface.u.legacy.tiling_index[0] = 14;
+                                break;
+			case 2:
+                                rtex->surface.u.legacy.tiling_index[0] = 15;
+                                break;
+			case 4:
+                                rtex->surface.u.legacy.tiling_index[0] = 16;
+                                break;
+			default: /* 8, 16 */
+                                rtex->surface.u.legacy.tiling_index[0] = 17;
+                                break;
+			}
+			break;
+		default: /* depth, thick */
+			assert(!"unexpected micro mode");
+			return;
+		}
+	}
+
+	rtex->surface.micro_tile_mode = rtex->last_msaa_resolve_target_micro_mode;
+
+	p_atomic_inc(&rscreen->dirty_tex_counter);
+}
+
+void evergreen_do_fast_color_clear(struct r600_common_context *rctx,
+				   struct pipe_framebuffer_state *fb,
+				   struct r600_atom *fb_state,
+				   unsigned *buffers, ubyte *dirty_cbufs,
+				   const union pipe_color_union *color)
+{
+	int i;
+
+	/* This function is broken in BE, so just disable this path for now */
+#ifdef PIPE_ARCH_BIG_ENDIAN
+	return;
+#endif
+
+	if (rctx->render_cond)
+		return;
+
+	for (i = 0; i < fb->nr_cbufs; i++) {
+		struct r600_texture *tex;
+		unsigned clear_bit = PIPE_CLEAR_COLOR0 << i;
+
+		if (!fb->cbufs[i])
+			continue;
+
+		/* if this colorbuffer is not being cleared */
+		if (!(*buffers & clear_bit))
+			continue;
+
+		tex = (struct r600_texture *)fb->cbufs[i]->texture;
+
+		/* the clear is allowed if all layers are bound */
+		if (fb->cbufs[i]->u.tex.first_layer != 0 ||
+		    fb->cbufs[i]->u.tex.last_layer != util_max_layer(&tex->resource.b.b, 0)) {
+			continue;
+		}
+
+		/* cannot clear mipmapped textures */
+		if (fb->cbufs[i]->texture->last_level != 0) {
+			continue;
+		}
+
+		/* only supported on tiled surfaces */
+		if (tex->surface.is_linear) {
+			continue;
+		}
+
+		/* shared textures can't use fast clear without an explicit flush,
+		 * because there is no way to communicate the clear color among
+		 * all clients
+		 */
+		if (tex->resource.b.is_shared &&
+		    !(tex->resource.external_usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH))
+			continue;
+
+		/* fast color clear with 1D tiling doesn't work on old kernels and CIK */
+		if (rctx->chip_class == CIK &&
+		    tex->surface.u.legacy.level[0].mode == RADEON_SURF_MODE_1D &&
+		    rctx->screen->info.drm_major == 2 &&
+		    rctx->screen->info.drm_minor < 38) {
+			continue;
+		}
+
+		{
+			/* 128-bit formats are unusupported */
+			if (tex->surface.bpe > 8) {
+				continue;
+			}
+
+			/* RB+ doesn't work with CMASK fast clear on Stoney. */
+			if (rctx->family == CHIP_STONEY)
+				continue;
+
+			/* ensure CMASK is enabled */
+			r600_texture_alloc_cmask_separate(rctx->screen, tex);
+			if (tex->cmask.size == 0) {
+				continue;
+			}
+
+			/* Do the fast clear. */
+			rctx->clear_buffer(&rctx->b, &tex->cmask_buffer->b.b,
+					   tex->cmask.offset, tex->cmask.size, 0,
+					   R600_COHERENCY_CB_META);
+
+			bool need_compressed_update = !tex->dirty_level_mask;
+
+			tex->dirty_level_mask |= 1 << fb->cbufs[i]->u.tex.level;
+
+			if (need_compressed_update)
+				p_atomic_inc(&rctx->screen->compressed_colortex_counter);
+		}
+
+		/* We can change the micro tile mode before a full clear. */
+		if (rctx->screen->chip_class >= SI)
+			si_set_optimal_micro_tile_mode(rctx->screen, tex);
+
+		evergreen_set_clear_color(tex, fb->cbufs[i]->format, color);
+
+		if (dirty_cbufs)
+			*dirty_cbufs |= 1 << i;
+		rctx->set_atom_dirty(rctx, fb_state, true);
+		*buffers &= ~clear_bit;
+	}
+}
+
+static struct pipe_memory_object *
+r600_memobj_from_handle(struct pipe_screen *screen,
+			struct winsys_handle *whandle,
+			bool dedicated)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+	struct r600_memory_object *memobj = CALLOC_STRUCT(r600_memory_object);
+	struct pb_buffer *buf = NULL;
+	uint32_t stride, offset;
+
+	if (!memobj)
+		return NULL;
+
+	buf = rscreen->ws->buffer_from_handle(rscreen->ws, whandle,
+					      &stride, &offset);
+	if (!buf) {
+		free(memobj);
+		return NULL;
+	}
+
+	memobj->b.dedicated = dedicated;
+	memobj->buf = buf;
+	memobj->stride = stride;
+	memobj->offset = offset;
+
+	return (struct pipe_memory_object *)memobj;
+
+}
+
+static void
+r600_memobj_destroy(struct pipe_screen *screen,
+		    struct pipe_memory_object *_memobj)
+{
+	struct r600_memory_object *memobj = (struct r600_memory_object *)_memobj;
+
+	pb_reference(&memobj->buf, NULL);
+	free(memobj);
+}
+
+static struct pipe_resource *
+r600_texture_from_memobj(struct pipe_screen *screen,
+			 const struct pipe_resource *templ,
+			 struct pipe_memory_object *_memobj,
+			 uint64_t offset)
+{
+	int r;
+	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+	struct r600_memory_object *memobj = (struct r600_memory_object *)_memobj;
+	struct r600_texture *rtex;
+	struct radeon_surf surface = {};
+	struct radeon_bo_metadata metadata = {};
+	enum radeon_surf_mode array_mode;
+	bool is_scanout;
+	struct pb_buffer *buf = NULL;
+
+	if (memobj->b.dedicated) {
+		rscreen->ws->buffer_get_metadata(memobj->buf, &metadata);
+		r600_surface_import_metadata(rscreen, &surface, &metadata,
+				     &array_mode, &is_scanout);
+	} else {
+		/**
+		 * The bo metadata is unset for un-dedicated images. So we fall
+		 * back to linear. See answer to question 5 of the
+		 * VK_KHX_external_memory spec for some details.
+		 *
+		 * It is possible that this case isn't going to work if the
+		 * surface pitch isn't correctly aligned by default.
+		 *
+		 * In order to support it correctly we require multi-image
+		 * metadata to be syncrhonized between radv and radeonsi. The
+		 * semantics of associating multiple image metadata to a memory
+		 * object on the vulkan export side are not concretely defined
+		 * either.
+		 *
+		 * All the use cases we are aware of at the moment for memory
+		 * objects use dedicated allocations. So lets keep the initial
+		 * implementation simple.
+		 *
+		 * A possible alternative is to attempt to reconstruct the
+		 * tiling information when the TexParameter TEXTURE_TILING_EXT
+		 * is set.
+		 */
+		array_mode = RADEON_SURF_MODE_LINEAR_ALIGNED;
+		is_scanout = false;
+
+	}
+
+	r = r600_init_surface(rscreen, &surface, templ,
+			      array_mode, memobj->stride,
+			      offset, true, is_scanout,
+			      false, false);
+	if (r)
+		return NULL;
+
+	rtex = r600_texture_create_object(screen, templ, memobj->buf, &surface);
+	if (!rtex)
+		return NULL;
+
+	/* r600_texture_create_object doesn't increment refcount of
+	 * memobj->buf, so increment it here.
+	 */
+	pb_reference(&buf, memobj->buf);
+
+	rtex->resource.b.is_shared = true;
+	rtex->resource.external_usage = PIPE_HANDLE_USAGE_READ_WRITE;
+
+	if (rscreen->apply_opaque_metadata)
+		rscreen->apply_opaque_metadata(rscreen, rtex, &metadata);
+
+	return &rtex->resource.b.b;
+}
+
+void r600_init_screen_texture_functions(struct r600_common_screen *rscreen)
+{
+	rscreen->b.resource_from_handle = r600_texture_from_handle;
+	rscreen->b.resource_get_handle = r600_texture_get_handle;
+	rscreen->b.resource_from_memobj = r600_texture_from_memobj;
+	rscreen->b.memobj_create_from_handle = r600_memobj_from_handle;
+	rscreen->b.memobj_destroy = r600_memobj_destroy;
+}
+
+void r600_init_context_texture_functions(struct r600_common_context *rctx)
+{
+	rctx->b.create_surface = r600_create_surface;
+	rctx->b.surface_destroy = r600_surface_destroy;
+	rctx->b.clear_texture = r600_clear_texture;
+}
diff --git a/src/gallium/drivers/r600/r600_uvd.c b/src/gallium/drivers/r600/r600_uvd.c
index cb732385d8b..28551e2daad 100644
--- a/src/gallium/drivers/r600/r600_uvd.c
+++ b/src/gallium/drivers/r600/r600_uvd.c
@@ -45,9 +45,9 @@
 #include "vl/vl_mpeg12_decoder.h"
 
 #include "r600_pipe.h"
-#include "radeon/radeon_video.h"
-#include "radeon/radeon_uvd.h"
-#include "radeon/radeon_vce.h"
+#include "radeon_video.h"
+#include "radeon_uvd.h"
+#include "radeon_vce.h"
 #include "r600d.h"
 
 #define R600_UVD_ENABLE_TILING 0
diff --git a/src/gallium/drivers/r600/r600_viewport.c b/src/gallium/drivers/r600/r600_viewport.c
new file mode 100644
index 00000000000..2de13820545
--- /dev/null
+++ b/src/gallium/drivers/r600/r600_viewport.c
@@ -0,0 +1,433 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "r600_cs.h"
+#include "util/u_viewport.h"
+#include "tgsi/tgsi_scan.h"
+
+#define GET_MAX_SCISSOR(rctx) (rctx->chip_class >= EVERGREEN ? 16384 : 8192)
+
+static void r600_set_scissor_states(struct pipe_context *ctx,
+				    unsigned start_slot,
+				    unsigned num_scissors,
+				    const struct pipe_scissor_state *state)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	int i;
+
+	for (i = 0; i < num_scissors; i++)
+		rctx->scissors.states[start_slot + i] = state[i];
+
+	if (!rctx->scissor_enabled)
+		return;
+
+	rctx->scissors.dirty_mask |= ((1 << num_scissors) - 1) << start_slot;
+	rctx->set_atom_dirty(rctx, &rctx->scissors.atom, true);
+}
+
+/* Since the guard band disables clipping, we have to clip per-pixel
+ * using a scissor.
+ */
+static void r600_get_scissor_from_viewport(struct r600_common_context *rctx,
+					   const struct pipe_viewport_state *vp,
+					   struct r600_signed_scissor *scissor)
+{
+	float tmp, minx, miny, maxx, maxy;
+
+	/* Convert (-1, -1) and (1, 1) from clip space into window space. */
+	minx = -vp->scale[0] + vp->translate[0];
+	miny = -vp->scale[1] + vp->translate[1];
+	maxx = vp->scale[0] + vp->translate[0];
+	maxy = vp->scale[1] + vp->translate[1];
+
+	/* r600_draw_rectangle sets this. Disable the scissor. */
+	if (minx == -1 && miny == -1 && maxx == 1 && maxy == 1) {
+		scissor->minx = scissor->miny = 0;
+		scissor->maxx = scissor->maxy = GET_MAX_SCISSOR(rctx);
+		return;
+	}
+
+	/* Handle inverted viewports. */
+	if (minx > maxx) {
+		tmp = minx;
+		minx = maxx;
+		maxx = tmp;
+	}
+	if (miny > maxy) {
+		tmp = miny;
+		miny = maxy;
+		maxy = tmp;
+	}
+
+	/* Convert to integer and round up the max bounds. */
+	scissor->minx = minx;
+	scissor->miny = miny;
+	scissor->maxx = ceilf(maxx);
+	scissor->maxy = ceilf(maxy);
+}
+
+static void r600_clamp_scissor(struct r600_common_context *rctx,
+			       struct pipe_scissor_state *out,
+			       struct r600_signed_scissor *scissor)
+{
+	unsigned max_scissor = GET_MAX_SCISSOR(rctx);
+	out->minx = CLAMP(scissor->minx, 0, max_scissor);
+	out->miny = CLAMP(scissor->miny, 0, max_scissor);
+	out->maxx = CLAMP(scissor->maxx, 0, max_scissor);
+	out->maxy = CLAMP(scissor->maxy, 0, max_scissor);
+}
+
+static void r600_clip_scissor(struct pipe_scissor_state *out,
+			      struct pipe_scissor_state *clip)
+{
+	out->minx = MAX2(out->minx, clip->minx);
+	out->miny = MAX2(out->miny, clip->miny);
+	out->maxx = MIN2(out->maxx, clip->maxx);
+	out->maxy = MIN2(out->maxy, clip->maxy);
+}
+
+static void r600_scissor_make_union(struct r600_signed_scissor *out,
+				    struct r600_signed_scissor *in)
+{
+	out->minx = MIN2(out->minx, in->minx);
+	out->miny = MIN2(out->miny, in->miny);
+	out->maxx = MAX2(out->maxx, in->maxx);
+	out->maxy = MAX2(out->maxy, in->maxy);
+}
+
+void evergreen_apply_scissor_bug_workaround(struct r600_common_context *rctx,
+					    struct pipe_scissor_state *scissor)
+{
+	if (rctx->chip_class == EVERGREEN || rctx->chip_class == CAYMAN) {
+		if (scissor->maxx == 0)
+			scissor->minx = 1;
+		if (scissor->maxy == 0)
+			scissor->miny = 1;
+
+		if (rctx->chip_class == CAYMAN &&
+		    scissor->maxx == 1 && scissor->maxy == 1)
+			scissor->maxx = 2;
+	}
+}
+
+static void r600_emit_one_scissor(struct r600_common_context *rctx,
+				  struct radeon_winsys_cs *cs,
+				  struct r600_signed_scissor *vp_scissor,
+				  struct pipe_scissor_state *scissor)
+{
+	struct pipe_scissor_state final;
+
+	if (rctx->vs_disables_clipping_viewport) {
+		final.minx = final.miny = 0;
+		final.maxx = final.maxy = GET_MAX_SCISSOR(rctx);
+	} else {
+		r600_clamp_scissor(rctx, &final, vp_scissor);
+	}
+
+	if (scissor)
+		r600_clip_scissor(&final, scissor);
+
+	evergreen_apply_scissor_bug_workaround(rctx, &final);
+
+	radeon_emit(cs, S_028250_TL_X(final.minx) |
+			S_028250_TL_Y(final.miny) |
+			S_028250_WINDOW_OFFSET_DISABLE(1));
+	radeon_emit(cs, S_028254_BR_X(final.maxx) |
+			S_028254_BR_Y(final.maxy));
+}
+
+/* the range is [-MAX, MAX] */
+#define GET_MAX_VIEWPORT_RANGE(rctx) (rctx->chip_class >= EVERGREEN ? 32768 : 16384)
+
+static void r600_emit_guardband(struct r600_common_context *rctx,
+				struct r600_signed_scissor *vp_as_scissor)
+{
+	struct radeon_winsys_cs *cs = rctx->gfx.cs;
+	struct pipe_viewport_state vp;
+	float left, top, right, bottom, max_range, guardband_x, guardband_y;
+
+	/* Reconstruct the viewport transformation from the scissor. */
+	vp.translate[0] = (vp_as_scissor->minx + vp_as_scissor->maxx) / 2.0;
+	vp.translate[1] = (vp_as_scissor->miny + vp_as_scissor->maxy) / 2.0;
+	vp.scale[0] = vp_as_scissor->maxx - vp.translate[0];
+	vp.scale[1] = vp_as_scissor->maxy - vp.translate[1];
+
+	/* Treat a 0x0 viewport as 1x1 to prevent division by zero. */
+	if (vp_as_scissor->minx == vp_as_scissor->maxx)
+		vp.scale[0] = 0.5;
+	if (vp_as_scissor->miny == vp_as_scissor->maxy)
+		vp.scale[1] = 0.5;
+
+	/* Find the biggest guard band that is inside the supported viewport
+	 * range. The guard band is specified as a horizontal and vertical
+	 * distance from (0,0) in clip space.
+	 *
+	 * This is done by applying the inverse viewport transformation
+	 * on the viewport limits to get those limits in clip space.
+	 *
+	 * Use a limit one pixel smaller to allow for some precision error.
+	 */
+	max_range = GET_MAX_VIEWPORT_RANGE(rctx) - 1;
+	left   = (-max_range - vp.translate[0]) / vp.scale[0];
+	right  = ( max_range - vp.translate[0]) / vp.scale[0];
+	top    = (-max_range - vp.translate[1]) / vp.scale[1];
+	bottom = ( max_range - vp.translate[1]) / vp.scale[1];
+
+	assert(left <= -1 && top <= -1 && right >= 1 && bottom >= 1);
+
+	guardband_x = MIN2(-left, right);
+	guardband_y = MIN2(-top, bottom);
+
+	/* If any of the GB registers is updated, all of them must be updated. */
+	if (rctx->chip_class >= CAYMAN)
+		radeon_set_context_reg_seq(cs, CM_R_028BE8_PA_CL_GB_VERT_CLIP_ADJ, 4);
+	else
+		radeon_set_context_reg_seq(cs, R600_R_028C0C_PA_CL_GB_VERT_CLIP_ADJ, 4);
+
+	radeon_emit(cs, fui(guardband_y)); /* R_028BE8_PA_CL_GB_VERT_CLIP_ADJ */
+	radeon_emit(cs, fui(1.0));         /* R_028BEC_PA_CL_GB_VERT_DISC_ADJ */
+	radeon_emit(cs, fui(guardband_x)); /* R_028BF0_PA_CL_GB_HORZ_CLIP_ADJ */
+	radeon_emit(cs, fui(1.0));         /* R_028BF4_PA_CL_GB_HORZ_DISC_ADJ */
+}
+
+static void r600_emit_scissors(struct r600_common_context *rctx, struct r600_atom *atom)
+{
+	struct radeon_winsys_cs *cs = rctx->gfx.cs;
+	struct pipe_scissor_state *states = rctx->scissors.states;
+	unsigned mask = rctx->scissors.dirty_mask;
+	bool scissor_enabled = rctx->scissor_enabled;
+	struct r600_signed_scissor max_vp_scissor;
+	int i;
+
+	/* The simple case: Only 1 viewport is active. */
+	if (!rctx->vs_writes_viewport_index) {
+		struct r600_signed_scissor *vp = &rctx->viewports.as_scissor[0];
+
+		if (!(mask & 1))
+			return;
+
+		radeon_set_context_reg_seq(cs, R_028250_PA_SC_VPORT_SCISSOR_0_TL, 2);
+		r600_emit_one_scissor(rctx, cs, vp, scissor_enabled ? &states[0] : NULL);
+		r600_emit_guardband(rctx, vp);
+		rctx->scissors.dirty_mask &= ~1; /* clear one bit */
+		return;
+	}
+
+	/* Shaders can draw to any viewport. Make a union of all viewports. */
+	max_vp_scissor = rctx->viewports.as_scissor[0];
+	for (i = 1; i < R600_MAX_VIEWPORTS; i++)
+		r600_scissor_make_union(&max_vp_scissor,
+				      &rctx->viewports.as_scissor[i]);
+
+	while (mask) {
+		int start, count, i;
+
+		u_bit_scan_consecutive_range(&mask, &start, &count);
+
+		radeon_set_context_reg_seq(cs, R_028250_PA_SC_VPORT_SCISSOR_0_TL +
+					       start * 4 * 2, count * 2);
+		for (i = start; i < start+count; i++) {
+			r600_emit_one_scissor(rctx, cs, &rctx->viewports.as_scissor[i],
+					      scissor_enabled ? &states[i] : NULL);
+		}
+	}
+	r600_emit_guardband(rctx, &max_vp_scissor);
+	rctx->scissors.dirty_mask = 0;
+}
+
+static void r600_set_viewport_states(struct pipe_context *ctx,
+				     unsigned start_slot,
+				     unsigned num_viewports,
+				     const struct pipe_viewport_state *state)
+{
+	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
+	unsigned mask;
+	int i;
+
+	for (i = 0; i < num_viewports; i++) {
+		unsigned index = start_slot + i;
+
+		rctx->viewports.states[index] = state[i];
+		r600_get_scissor_from_viewport(rctx, &state[i],
+					       &rctx->viewports.as_scissor[index]);
+	}
+
+	mask = ((1 << num_viewports) - 1) << start_slot;
+	rctx->viewports.dirty_mask |= mask;
+	rctx->viewports.depth_range_dirty_mask |= mask;
+	rctx->scissors.dirty_mask |= mask;
+	rctx->set_atom_dirty(rctx, &rctx->viewports.atom, true);
+	rctx->set_atom_dirty(rctx, &rctx->scissors.atom, true);
+}
+
+static void r600_emit_one_viewport(struct r600_common_context *rctx,
+				   struct pipe_viewport_state *state)
+{
+	struct radeon_winsys_cs *cs = rctx->gfx.cs;
+
+	radeon_emit(cs, fui(state->scale[0]));
+	radeon_emit(cs, fui(state->translate[0]));
+	radeon_emit(cs, fui(state->scale[1]));
+	radeon_emit(cs, fui(state->translate[1]));
+	radeon_emit(cs, fui(state->scale[2]));
+	radeon_emit(cs, fui(state->translate[2]));
+}
+
+static void r600_emit_viewports(struct r600_common_context *rctx)
+{
+	struct radeon_winsys_cs *cs = rctx->gfx.cs;
+	struct pipe_viewport_state *states = rctx->viewports.states;
+	unsigned mask = rctx->viewports.dirty_mask;
+
+	/* The simple case: Only 1 viewport is active. */
+	if (!rctx->vs_writes_viewport_index) {
+		if (!(mask & 1))
+			return;
+
+		radeon_set_context_reg_seq(cs, R_02843C_PA_CL_VPORT_XSCALE, 6);
+		r600_emit_one_viewport(rctx, &states[0]);
+		rctx->viewports.dirty_mask &= ~1; /* clear one bit */
+		return;
+	}
+
+	while (mask) {
+		int start, count, i;
+
+		u_bit_scan_consecutive_range(&mask, &start, &count);
+
+		radeon_set_context_reg_seq(cs, R_02843C_PA_CL_VPORT_XSCALE +
+					       start * 4 * 6, count * 6);
+		for (i = start; i < start+count; i++)
+			r600_emit_one_viewport(rctx, &states[i]);
+	}
+	rctx->viewports.dirty_mask = 0;
+}
+
+static void r600_emit_depth_ranges(struct r600_common_context *rctx)
+{
+	struct radeon_winsys_cs *cs = rctx->gfx.cs;
+	struct pipe_viewport_state *states = rctx->viewports.states;
+	unsigned mask = rctx->viewports.depth_range_dirty_mask;
+	float zmin, zmax;
+
+	/* The simple case: Only 1 viewport is active. */
+	if (!rctx->vs_writes_viewport_index) {
+		if (!(mask & 1))
+			return;
+
+		util_viewport_zmin_zmax(&states[0], rctx->clip_halfz, &zmin, &zmax);
+
+		radeon_set_context_reg_seq(cs, R_0282D0_PA_SC_VPORT_ZMIN_0, 2);
+		radeon_emit(cs, fui(zmin));
+		radeon_emit(cs, fui(zmax));
+		rctx->viewports.depth_range_dirty_mask &= ~1; /* clear one bit */
+		return;
+	}
+
+	while (mask) {
+		int start, count, i;
+
+		u_bit_scan_consecutive_range(&mask, &start, &count);
+
+		radeon_set_context_reg_seq(cs, R_0282D0_PA_SC_VPORT_ZMIN_0 +
+					   start * 4 * 2, count * 2);
+		for (i = start; i < start+count; i++) {
+			util_viewport_zmin_zmax(&states[i], rctx->clip_halfz, &zmin, &zmax);
+			radeon_emit(cs, fui(zmin));
+			radeon_emit(cs, fui(zmax));
+		}
+	}
+	rctx->viewports.depth_range_dirty_mask = 0;
+}
+
+static void r600_emit_viewport_states(struct r600_common_context *rctx,
+				      struct r600_atom *atom)
+{
+	r600_emit_viewports(rctx);
+	r600_emit_depth_ranges(rctx);
+}
+
+/* Set viewport dependencies on pipe_rasterizer_state. */
+void r600_viewport_set_rast_deps(struct r600_common_context *rctx,
+				 bool scissor_enable, bool clip_halfz)
+{
+	if (rctx->scissor_enabled != scissor_enable) {
+		rctx->scissor_enabled = scissor_enable;
+		rctx->scissors.dirty_mask = (1 << R600_MAX_VIEWPORTS) - 1;
+		rctx->set_atom_dirty(rctx, &rctx->scissors.atom, true);
+	}
+	if (rctx->clip_halfz != clip_halfz) {
+		rctx->clip_halfz = clip_halfz;
+		rctx->viewports.depth_range_dirty_mask = (1 << R600_MAX_VIEWPORTS) - 1;
+		rctx->set_atom_dirty(rctx, &rctx->viewports.atom, true);
+	}
+}
+
+/**
+ * Normally, we only emit 1 viewport and 1 scissor if no shader is using
+ * the VIEWPORT_INDEX output, and emitting the other viewports and scissors
+ * is delayed. When a shader with VIEWPORT_INDEX appears, this should be
+ * called to emit the rest.
+ */
+void r600_update_vs_writes_viewport_index(struct r600_common_context *rctx,
+					  struct tgsi_shader_info *info)
+{
+	bool vs_window_space;
+
+	if (!info)
+		return;
+
+	/* When the VS disables clipping and viewport transformation. */
+	vs_window_space =
+		info->properties[TGSI_PROPERTY_VS_WINDOW_SPACE_POSITION];
+
+	if (rctx->vs_disables_clipping_viewport != vs_window_space) {
+		rctx->vs_disables_clipping_viewport = vs_window_space;
+		rctx->scissors.dirty_mask = (1 << R600_MAX_VIEWPORTS) - 1;
+		rctx->set_atom_dirty(rctx, &rctx->scissors.atom, true);
+	}
+
+	/* Viewport index handling. */
+	rctx->vs_writes_viewport_index = info->writes_viewport_index;
+	if (!rctx->vs_writes_viewport_index)
+		return;
+
+	if (rctx->scissors.dirty_mask)
+	    rctx->set_atom_dirty(rctx, &rctx->scissors.atom, true);
+
+	if (rctx->viewports.dirty_mask ||
+	    rctx->viewports.depth_range_dirty_mask)
+	    rctx->set_atom_dirty(rctx, &rctx->viewports.atom, true);
+}
+
+void r600_init_viewport_functions(struct r600_common_context *rctx)
+{
+	rctx->scissors.atom.emit = r600_emit_scissors;
+	rctx->viewports.atom.emit = r600_emit_viewport_states;
+
+	rctx->scissors.atom.num_dw = (2 + 16 * 2) + 6;
+	rctx->viewports.atom.num_dw = 2 + 16 * 6;
+
+	rctx->b.set_scissor_states = r600_set_scissor_states;
+	rctx->b.set_viewport_states = r600_set_viewport_states;
+}
diff --git a/src/gallium/drivers/r600/radeon_uvd.c b/src/gallium/drivers/r600/radeon_uvd.c
new file mode 100644
index 00000000000..fabc73e3834
--- /dev/null
+++ b/src/gallium/drivers/r600/radeon_uvd.c
@@ -0,0 +1,1618 @@
+/**************************************************************************
+ *
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+/*
+ * Authors:
+ *	Christian König <[email protected]>
+ *
+ */
+
+#include <sys/types.h>
+#include <assert.h>
+#include <errno.h>
+#include <unistd.h>
+#include <stdio.h>
+
+#include "pipe/p_video_codec.h"
+
+#include "util/u_memory.h"
+#include "util/u_video.h"
+
+#include "vl/vl_defines.h"
+#include "vl/vl_mpeg12_decoder.h"
+
+#include "r600_pipe_common.h"
+#include "radeon_video.h"
+#include "radeon_uvd.h"
+
+#define NUM_BUFFERS 4
+
+#define NUM_MPEG2_REFS 6
+#define NUM_H264_REFS 17
+#define NUM_VC1_REFS 5
+
+#define FB_BUFFER_OFFSET 0x1000
+#define FB_BUFFER_SIZE 2048
+#define FB_BUFFER_SIZE_TONGA (2048 * 64)
+#define IT_SCALING_TABLE_SIZE 992
+#define UVD_SESSION_CONTEXT_SIZE (128 * 1024)
+
+/* UVD decoder representation */
+struct ruvd_decoder {
+	struct pipe_video_codec		base;
+
+	ruvd_set_dtb			set_dtb;
+
+	unsigned			stream_handle;
+	unsigned			stream_type;
+	unsigned			frame_number;
+
+	struct pipe_screen		*screen;
+	struct radeon_winsys*		ws;
+	struct radeon_winsys_cs*	cs;
+
+	unsigned			cur_buffer;
+
+	struct rvid_buffer		msg_fb_it_buffers[NUM_BUFFERS];
+	struct ruvd_msg			*msg;
+	uint32_t			*fb;
+	unsigned			fb_size;
+	uint8_t				*it;
+
+	struct rvid_buffer		bs_buffers[NUM_BUFFERS];
+	void*				bs_ptr;
+	unsigned			bs_size;
+
+	struct rvid_buffer		dpb;
+	bool				use_legacy;
+	struct rvid_buffer		ctx;
+	struct rvid_buffer		sessionctx;
+	struct {
+		unsigned 		data0;
+		unsigned		data1;
+		unsigned		cmd;
+		unsigned		cntl;
+	} reg;
+};
+
+/* flush IB to the hardware */
+static int flush(struct ruvd_decoder *dec, unsigned flags)
+{
+	return dec->ws->cs_flush(dec->cs, flags, NULL);
+}
+
+/* add a new set register command to the IB */
+static void set_reg(struct ruvd_decoder *dec, unsigned reg, uint32_t val)
+{
+	radeon_emit(dec->cs, RUVD_PKT0(reg >> 2, 0));
+	radeon_emit(dec->cs, val);
+}
+
+/* send a command to the VCPU through the GPCOM registers */
+static void send_cmd(struct ruvd_decoder *dec, unsigned cmd,
+		     struct pb_buffer* buf, uint32_t off,
+		     enum radeon_bo_usage usage, enum radeon_bo_domain domain)
+{
+	int reloc_idx;
+
+	reloc_idx = dec->ws->cs_add_buffer(dec->cs, buf, usage | RADEON_USAGE_SYNCHRONIZED,
+					   domain,
+					  RADEON_PRIO_UVD);
+	if (!dec->use_legacy) {
+		uint64_t addr;
+		addr = dec->ws->buffer_get_virtual_address(buf);
+		addr = addr + off;
+		set_reg(dec, dec->reg.data0, addr);
+		set_reg(dec, dec->reg.data1, addr >> 32);
+	} else {
+		off += dec->ws->buffer_get_reloc_offset(buf);
+		set_reg(dec, RUVD_GPCOM_VCPU_DATA0, off);
+		set_reg(dec, RUVD_GPCOM_VCPU_DATA1, reloc_idx * 4);
+	}
+	set_reg(dec, dec->reg.cmd, cmd << 1);
+}
+
+/* do the codec needs an IT buffer ?*/
+static bool have_it(struct ruvd_decoder *dec)
+{
+	return dec->stream_type == RUVD_CODEC_H264_PERF ||
+		dec->stream_type == RUVD_CODEC_H265;
+}
+
+/* map the next available message/feedback/itscaling buffer */
+static void map_msg_fb_it_buf(struct ruvd_decoder *dec)
+{
+	struct rvid_buffer* buf;
+	uint8_t *ptr;
+
+	/* grab the current message/feedback buffer */
+	buf = &dec->msg_fb_it_buffers[dec->cur_buffer];
+
+	/* and map it for CPU access */
+	ptr = dec->ws->buffer_map(buf->res->buf, dec->cs, PIPE_TRANSFER_WRITE);
+
+	/* calc buffer offsets */
+	dec->msg = (struct ruvd_msg *)ptr;
+	memset(dec->msg, 0, sizeof(*dec->msg));
+
+	dec->fb = (uint32_t *)(ptr + FB_BUFFER_OFFSET);
+	if (have_it(dec))
+		dec->it = (uint8_t *)(ptr + FB_BUFFER_OFFSET + dec->fb_size);
+}
+
+/* unmap and send a message command to the VCPU */
+static void send_msg_buf(struct ruvd_decoder *dec)
+{
+	struct rvid_buffer* buf;
+
+	/* ignore the request if message/feedback buffer isn't mapped */
+	if (!dec->msg || !dec->fb)
+		return;
+
+	/* grab the current message buffer */
+	buf = &dec->msg_fb_it_buffers[dec->cur_buffer];
+
+	/* unmap the buffer */
+	dec->ws->buffer_unmap(buf->res->buf);
+	dec->msg = NULL;
+	dec->fb = NULL;
+	dec->it = NULL;
+
+
+	if (dec->sessionctx.res)
+		send_cmd(dec, RUVD_CMD_SESSION_CONTEXT_BUFFER,
+			 dec->sessionctx.res->buf, 0, RADEON_USAGE_READWRITE,
+			 RADEON_DOMAIN_VRAM);
+
+	/* and send it to the hardware */
+	send_cmd(dec, RUVD_CMD_MSG_BUFFER, buf->res->buf, 0,
+		 RADEON_USAGE_READ, RADEON_DOMAIN_GTT);
+}
+
+/* cycle to the next set of buffers */
+static void next_buffer(struct ruvd_decoder *dec)
+{
+	++dec->cur_buffer;
+	dec->cur_buffer %= NUM_BUFFERS;
+}
+
+/* convert the profile into something UVD understands */
+static uint32_t profile2stream_type(struct ruvd_decoder *dec, unsigned family)
+{
+	switch (u_reduce_video_profile(dec->base.profile)) {
+	case PIPE_VIDEO_FORMAT_MPEG4_AVC:
+		return (family >= CHIP_TONGA) ?
+			RUVD_CODEC_H264_PERF : RUVD_CODEC_H264;
+
+	case PIPE_VIDEO_FORMAT_VC1:
+		return RUVD_CODEC_VC1;
+
+	case PIPE_VIDEO_FORMAT_MPEG12:
+		return RUVD_CODEC_MPEG2;
+
+	case PIPE_VIDEO_FORMAT_MPEG4:
+		return RUVD_CODEC_MPEG4;
+
+	case PIPE_VIDEO_FORMAT_HEVC:
+		return RUVD_CODEC_H265;
+
+	case PIPE_VIDEO_FORMAT_JPEG:
+		return RUVD_CODEC_MJPEG;
+
+	default:
+		assert(0);
+		return 0;
+	}
+}
+
+static unsigned calc_ctx_size_h264_perf(struct ruvd_decoder *dec)
+{
+	unsigned width_in_mb, height_in_mb, ctx_size;
+	unsigned width = align(dec->base.width, VL_MACROBLOCK_WIDTH);
+	unsigned height = align(dec->base.height, VL_MACROBLOCK_HEIGHT);
+
+	unsigned max_references = dec->base.max_references + 1;
+
+	// picture width & height in 16 pixel units
+	width_in_mb = width / VL_MACROBLOCK_WIDTH;
+	height_in_mb = align(height / VL_MACROBLOCK_HEIGHT, 2);
+
+	if (!dec->use_legacy) {
+		unsigned fs_in_mb = width_in_mb * height_in_mb;
+		unsigned num_dpb_buffer;
+		switch(dec->base.level) {
+		case 30:
+			num_dpb_buffer = 8100 / fs_in_mb;
+			break;
+		case 31:
+			num_dpb_buffer = 18000 / fs_in_mb;
+			break;
+		case 32:
+			num_dpb_buffer = 20480 / fs_in_mb;
+			break;
+		case 41:
+			num_dpb_buffer = 32768 / fs_in_mb;
+			break;
+		case 42:
+			num_dpb_buffer = 34816 / fs_in_mb;
+			break;
+		case 50:
+			num_dpb_buffer = 110400 / fs_in_mb;
+			break;
+		case 51:
+			num_dpb_buffer = 184320 / fs_in_mb;
+			break;
+		default:
+			num_dpb_buffer = 184320 / fs_in_mb;
+			break;
+		}
+		num_dpb_buffer++;
+		max_references = MAX2(MIN2(NUM_H264_REFS, num_dpb_buffer), max_references);
+		ctx_size = max_references * align(width_in_mb * height_in_mb  * 192, 256);
+	} else {
+		// the firmware seems to always assume a minimum of ref frames
+		max_references = MAX2(NUM_H264_REFS, max_references);
+		// macroblock context buffer
+		ctx_size = align(width_in_mb * height_in_mb * max_references * 192, 256);
+	}
+
+	return ctx_size;
+}
+
+static unsigned calc_ctx_size_h265_main(struct ruvd_decoder *dec)
+{
+	unsigned width = align(dec->base.width, VL_MACROBLOCK_WIDTH);
+	unsigned height = align(dec->base.height, VL_MACROBLOCK_HEIGHT);
+
+	unsigned max_references = dec->base.max_references + 1;
+
+	if (dec->base.width * dec->base.height >= 4096*2000)
+		max_references = MAX2(max_references, 8);
+	else
+		max_references = MAX2(max_references, 17);
+
+	width = align (width, 16);
+	height = align (height, 16);
+	return ((width + 255) / 16) * ((height + 255) / 16) * 16 * max_references + 52 * 1024;
+}
+
+static unsigned calc_ctx_size_h265_main10(struct ruvd_decoder *dec, struct pipe_h265_picture_desc *pic)
+{
+	unsigned block_size, log2_ctb_size, width_in_ctb, height_in_ctb, num_16x16_block_per_ctb;
+	unsigned context_buffer_size_per_ctb_row, cm_buffer_size, max_mb_address, db_left_tile_pxl_size;
+	unsigned db_left_tile_ctx_size = 4096 / 16 * (32 + 16 * 4);
+
+	unsigned width = align(dec->base.width, VL_MACROBLOCK_WIDTH);
+	unsigned height = align(dec->base.height, VL_MACROBLOCK_HEIGHT);
+	unsigned coeff_10bit = (pic->pps->sps->bit_depth_luma_minus8 || pic->pps->sps->bit_depth_chroma_minus8) ? 2 : 1;
+
+	unsigned max_references = dec->base.max_references + 1;
+
+	if (dec->base.width * dec->base.height >= 4096*2000)
+		max_references = MAX2(max_references, 8);
+	else
+		max_references = MAX2(max_references, 17);
+
+	block_size = (1 << (pic->pps->sps->log2_min_luma_coding_block_size_minus3 + 3));
+	log2_ctb_size = block_size + pic->pps->sps->log2_diff_max_min_luma_coding_block_size;
+
+	width_in_ctb = (width + ((1 << log2_ctb_size) - 1)) >> log2_ctb_size;
+	height_in_ctb = (height + ((1 << log2_ctb_size) - 1)) >> log2_ctb_size;
+
+	num_16x16_block_per_ctb = ((1 << log2_ctb_size) >> 4) * ((1 << log2_ctb_size) >> 4);
+	context_buffer_size_per_ctb_row = align(width_in_ctb * num_16x16_block_per_ctb * 16, 256);
+	max_mb_address = (unsigned) ceil(height * 8 / 2048.0);
+
+	cm_buffer_size = max_references * context_buffer_size_per_ctb_row * height_in_ctb;
+	db_left_tile_pxl_size = coeff_10bit * (max_mb_address * 2 * 2048 + 1024);
+
+	return cm_buffer_size + db_left_tile_ctx_size + db_left_tile_pxl_size;
+}
+
+static unsigned get_db_pitch_alignment(struct ruvd_decoder *dec)
+{
+	if (((struct r600_common_screen*)dec->screen)->family < CHIP_VEGA10)
+		return 16;
+	else
+		return 32;
+}
+
+/* calculate size of reference picture buffer */
+static unsigned calc_dpb_size(struct ruvd_decoder *dec)
+{
+	unsigned width_in_mb, height_in_mb, image_size, dpb_size;
+
+	// always align them to MB size for dpb calculation
+	unsigned width = align(dec->base.width, VL_MACROBLOCK_WIDTH);
+	unsigned height = align(dec->base.height, VL_MACROBLOCK_HEIGHT);
+
+	// always one more for currently decoded picture
+	unsigned max_references = dec->base.max_references + 1;
+
+	// aligned size of a single frame
+	image_size = align(width, get_db_pitch_alignment(dec)) * height;
+	image_size += image_size / 2;
+	image_size = align(image_size, 1024);
+
+	// picture width & height in 16 pixel units
+	width_in_mb = width / VL_MACROBLOCK_WIDTH;
+	height_in_mb = align(height / VL_MACROBLOCK_HEIGHT, 2);
+
+	switch (u_reduce_video_profile(dec->base.profile)) {
+	case PIPE_VIDEO_FORMAT_MPEG4_AVC: {
+		if (!dec->use_legacy) {
+			unsigned fs_in_mb = width_in_mb * height_in_mb;
+			unsigned alignment = 64, num_dpb_buffer;
+
+			if (dec->stream_type == RUVD_CODEC_H264_PERF)
+				alignment = 256;
+			switch(dec->base.level) {
+			case 30:
+				num_dpb_buffer = 8100 / fs_in_mb;
+				break;
+			case 31:
+				num_dpb_buffer = 18000 / fs_in_mb;
+				break;
+			case 32:
+				num_dpb_buffer = 20480 / fs_in_mb;
+				break;
+			case 41:
+				num_dpb_buffer = 32768 / fs_in_mb;
+				break;
+			case 42:
+				num_dpb_buffer = 34816 / fs_in_mb;
+				break;
+			case 50:
+				num_dpb_buffer = 110400 / fs_in_mb;
+				break;
+			case 51:
+				num_dpb_buffer = 184320 / fs_in_mb;
+				break;
+			default:
+				num_dpb_buffer = 184320 / fs_in_mb;
+				break;
+			}
+			num_dpb_buffer++;
+			max_references = MAX2(MIN2(NUM_H264_REFS, num_dpb_buffer), max_references);
+			dpb_size = image_size * max_references;
+			if ((dec->stream_type != RUVD_CODEC_H264_PERF) ||
+			    (((struct r600_common_screen*)dec->screen)->family < CHIP_POLARIS10)) {
+				dpb_size += max_references * align(width_in_mb * height_in_mb  * 192, alignment);
+				dpb_size += align(width_in_mb * height_in_mb * 32, alignment);
+			}
+		} else {
+			// the firmware seems to allways assume a minimum of ref frames
+			max_references = MAX2(NUM_H264_REFS, max_references);
+			// reference picture buffer
+			dpb_size = image_size * max_references;
+			if ((dec->stream_type != RUVD_CODEC_H264_PERF) ||
+			    (((struct r600_common_screen*)dec->screen)->family < CHIP_POLARIS10)) {
+				// macroblock context buffer
+				dpb_size += width_in_mb * height_in_mb * max_references * 192;
+				// IT surface buffer
+				dpb_size += width_in_mb * height_in_mb * 32;
+			}
+		}
+		break;
+	}
+
+	case PIPE_VIDEO_FORMAT_HEVC:
+		if (dec->base.width * dec->base.height >= 4096*2000)
+			max_references = MAX2(max_references, 8);
+		else
+			max_references = MAX2(max_references, 17);
+
+		width = align (width, 16);
+		height = align (height, 16);
+		if (dec->base.profile == PIPE_VIDEO_PROFILE_HEVC_MAIN_10)
+			dpb_size = align((align(width, get_db_pitch_alignment(dec)) * height * 9) / 4, 256) * max_references;
+		else
+			dpb_size = align((align(width, get_db_pitch_alignment(dec)) * height * 3) / 2, 256) * max_references;
+		break;
+
+	case PIPE_VIDEO_FORMAT_VC1:
+		// the firmware seems to allways assume a minimum of ref frames
+		max_references = MAX2(NUM_VC1_REFS, max_references);
+
+		// reference picture buffer
+		dpb_size = image_size * max_references;
+
+		// CONTEXT_BUFFER
+		dpb_size += width_in_mb * height_in_mb * 128;
+
+		// IT surface buffer
+		dpb_size += width_in_mb * 64;
+
+		// DB surface buffer
+		dpb_size += width_in_mb * 128;
+
+		// BP
+		dpb_size += align(MAX2(width_in_mb, height_in_mb) * 7 * 16, 64);
+		break;
+
+	case PIPE_VIDEO_FORMAT_MPEG12:
+		// reference picture buffer, must be big enough for all frames
+		dpb_size = image_size * NUM_MPEG2_REFS;
+		break;
+
+	case PIPE_VIDEO_FORMAT_MPEG4:
+		// reference picture buffer
+		dpb_size = image_size * max_references;
+
+		// CM
+		dpb_size += width_in_mb * height_in_mb * 64;
+
+		// IT surface buffer
+		dpb_size += align(width_in_mb * height_in_mb * 32, 64);
+
+		dpb_size = MAX2(dpb_size, 30 * 1024 * 1024);
+		break;
+
+	case PIPE_VIDEO_FORMAT_JPEG:
+		dpb_size = 0;
+		break;
+
+	default:
+		// something is missing here
+		assert(0);
+
+		// at least use a sane default value
+		dpb_size = 32 * 1024 * 1024;
+		break;
+	}
+	return dpb_size;
+}
+
+/* free associated data in the video buffer callback */
+static void ruvd_destroy_associated_data(void *data)
+{
+	/* NOOP, since we only use an intptr */
+}
+
+/* get h264 specific message bits */
+static struct ruvd_h264 get_h264_msg(struct ruvd_decoder *dec, struct pipe_h264_picture_desc *pic)
+{
+	struct ruvd_h264 result;
+
+	memset(&result, 0, sizeof(result));
+	switch (pic->base.profile) {
+	case PIPE_VIDEO_PROFILE_MPEG4_AVC_BASELINE:
+	case PIPE_VIDEO_PROFILE_MPEG4_AVC_CONSTRAINED_BASELINE:
+		result.profile = RUVD_H264_PROFILE_BASELINE;
+		break;
+
+	case PIPE_VIDEO_PROFILE_MPEG4_AVC_MAIN:
+		result.profile = RUVD_H264_PROFILE_MAIN;
+		break;
+
+	case PIPE_VIDEO_PROFILE_MPEG4_AVC_HIGH:
+		result.profile = RUVD_H264_PROFILE_HIGH;
+		break;
+
+	default:
+		assert(0);
+		break;
+	}
+
+	result.level = dec->base.level;
+
+	result.sps_info_flags = 0;
+	result.sps_info_flags |= pic->pps->sps->direct_8x8_inference_flag << 0;
+	result.sps_info_flags |= pic->pps->sps->mb_adaptive_frame_field_flag << 1;
+	result.sps_info_flags |= pic->pps->sps->frame_mbs_only_flag << 2;
+	result.sps_info_flags |= pic->pps->sps->delta_pic_order_always_zero_flag << 3;
+
+	result.bit_depth_luma_minus8 = pic->pps->sps->bit_depth_luma_minus8;
+	result.bit_depth_chroma_minus8 = pic->pps->sps->bit_depth_chroma_minus8;
+	result.log2_max_frame_num_minus4 = pic->pps->sps->log2_max_frame_num_minus4;
+	result.pic_order_cnt_type = pic->pps->sps->pic_order_cnt_type;
+	result.log2_max_pic_order_cnt_lsb_minus4 = pic->pps->sps->log2_max_pic_order_cnt_lsb_minus4;
+
+	switch (dec->base.chroma_format) {
+	case PIPE_VIDEO_CHROMA_FORMAT_NONE:
+		/* TODO: assert? */
+		break;
+	case PIPE_VIDEO_CHROMA_FORMAT_400:
+		result.chroma_format = 0;
+		break;
+	case PIPE_VIDEO_CHROMA_FORMAT_420:
+		result.chroma_format = 1;
+		break;
+	case PIPE_VIDEO_CHROMA_FORMAT_422:
+		result.chroma_format = 2;
+		break;
+	case PIPE_VIDEO_CHROMA_FORMAT_444:
+		result.chroma_format = 3;
+		break;
+	}
+
+	result.pps_info_flags = 0;
+	result.pps_info_flags |= pic->pps->transform_8x8_mode_flag << 0;
+	result.pps_info_flags |= pic->pps->redundant_pic_cnt_present_flag << 1;
+	result.pps_info_flags |= pic->pps->constrained_intra_pred_flag << 2;
+	result.pps_info_flags |= pic->pps->deblocking_filter_control_present_flag << 3;
+	result.pps_info_flags |= pic->pps->weighted_bipred_idc << 4;
+	result.pps_info_flags |= pic->pps->weighted_pred_flag << 6;
+	result.pps_info_flags |= pic->pps->bottom_field_pic_order_in_frame_present_flag << 7;
+	result.pps_info_flags |= pic->pps->entropy_coding_mode_flag << 8;
+
+	result.num_slice_groups_minus1 = pic->pps->num_slice_groups_minus1;
+	result.slice_group_map_type = pic->pps->slice_group_map_type;
+	result.slice_group_change_rate_minus1 = pic->pps->slice_group_change_rate_minus1;
+	result.pic_init_qp_minus26 = pic->pps->pic_init_qp_minus26;
+	result.chroma_qp_index_offset = pic->pps->chroma_qp_index_offset;
+	result.second_chroma_qp_index_offset = pic->pps->second_chroma_qp_index_offset;
+
+	memcpy(result.scaling_list_4x4, pic->pps->ScalingList4x4, 6*16);
+	memcpy(result.scaling_list_8x8, pic->pps->ScalingList8x8, 2*64);
+
+	if (dec->stream_type == RUVD_CODEC_H264_PERF) {
+		memcpy(dec->it, result.scaling_list_4x4, 6*16);
+		memcpy((dec->it + 96), result.scaling_list_8x8, 2*64);
+	}
+
+	result.num_ref_frames = pic->num_ref_frames;
+
+	result.num_ref_idx_l0_active_minus1 = pic->num_ref_idx_l0_active_minus1;
+	result.num_ref_idx_l1_active_minus1 = pic->num_ref_idx_l1_active_minus1;
+
+	result.frame_num = pic->frame_num;
+	memcpy(result.frame_num_list, pic->frame_num_list, 4*16);
+	result.curr_field_order_cnt_list[0] = pic->field_order_cnt[0];
+	result.curr_field_order_cnt_list[1] = pic->field_order_cnt[1];
+	memcpy(result.field_order_cnt_list, pic->field_order_cnt_list, 4*16*2);
+
+	result.decoded_pic_idx = pic->frame_num;
+
+	return result;
+}
+
+/* get h265 specific message bits */
+static struct ruvd_h265 get_h265_msg(struct ruvd_decoder *dec, struct pipe_video_buffer *target,
+				     struct pipe_h265_picture_desc *pic)
+{
+	struct ruvd_h265 result;
+	unsigned i;
+
+	memset(&result, 0, sizeof(result));
+
+	result.sps_info_flags = 0;
+	result.sps_info_flags |= pic->pps->sps->scaling_list_enabled_flag << 0;
+	result.sps_info_flags |= pic->pps->sps->amp_enabled_flag << 1;
+	result.sps_info_flags |= pic->pps->sps->sample_adaptive_offset_enabled_flag << 2;
+	result.sps_info_flags |= pic->pps->sps->pcm_enabled_flag << 3;
+	result.sps_info_flags |= pic->pps->sps->pcm_loop_filter_disabled_flag << 4;
+	result.sps_info_flags |= pic->pps->sps->long_term_ref_pics_present_flag << 5;
+	result.sps_info_flags |= pic->pps->sps->sps_temporal_mvp_enabled_flag << 6;
+	result.sps_info_flags |= pic->pps->sps->strong_intra_smoothing_enabled_flag << 7;
+	result.sps_info_flags |= pic->pps->sps->separate_colour_plane_flag << 8;
+	if (((struct r600_common_screen*)dec->screen)->family == CHIP_CARRIZO)
+		result.sps_info_flags |= 1 << 9;
+	if (pic->UseRefPicList == true)
+		result.sps_info_flags |= 1 << 10;
+
+	result.chroma_format = pic->pps->sps->chroma_format_idc;
+	result.bit_depth_luma_minus8 = pic->pps->sps->bit_depth_luma_minus8;
+	result.bit_depth_chroma_minus8 = pic->pps->sps->bit_depth_chroma_minus8;
+	result.log2_max_pic_order_cnt_lsb_minus4 = pic->pps->sps->log2_max_pic_order_cnt_lsb_minus4;
+	result.sps_max_dec_pic_buffering_minus1 = pic->pps->sps->sps_max_dec_pic_buffering_minus1;
+	result.log2_min_luma_coding_block_size_minus3 = pic->pps->sps->log2_min_luma_coding_block_size_minus3;
+	result.log2_diff_max_min_luma_coding_block_size = pic->pps->sps->log2_diff_max_min_luma_coding_block_size;
+	result.log2_min_transform_block_size_minus2 = pic->pps->sps->log2_min_transform_block_size_minus2;
+	result.log2_diff_max_min_transform_block_size = pic->pps->sps->log2_diff_max_min_transform_block_size;
+	result.max_transform_hierarchy_depth_inter = pic->pps->sps->max_transform_hierarchy_depth_inter;
+	result.max_transform_hierarchy_depth_intra = pic->pps->sps->max_transform_hierarchy_depth_intra;
+	result.pcm_sample_bit_depth_luma_minus1 = pic->pps->sps->pcm_sample_bit_depth_luma_minus1;
+	result.pcm_sample_bit_depth_chroma_minus1 = pic->pps->sps->pcm_sample_bit_depth_chroma_minus1;
+	result.log2_min_pcm_luma_coding_block_size_minus3 = pic->pps->sps->log2_min_pcm_luma_coding_block_size_minus3;
+	result.log2_diff_max_min_pcm_luma_coding_block_size = pic->pps->sps->log2_diff_max_min_pcm_luma_coding_block_size;
+	result.num_short_term_ref_pic_sets = pic->pps->sps->num_short_term_ref_pic_sets;
+
+	result.pps_info_flags = 0;
+	result.pps_info_flags |= pic->pps->dependent_slice_segments_enabled_flag << 0;
+	result.pps_info_flags |= pic->pps->output_flag_present_flag << 1;
+	result.pps_info_flags |= pic->pps->sign_data_hiding_enabled_flag << 2;
+	result.pps_info_flags |= pic->pps->cabac_init_present_flag << 3;
+	result.pps_info_flags |= pic->pps->constrained_intra_pred_flag << 4;
+	result.pps_info_flags |= pic->pps->transform_skip_enabled_flag << 5;
+	result.pps_info_flags |= pic->pps->cu_qp_delta_enabled_flag << 6;
+	result.pps_info_flags |= pic->pps->pps_slice_chroma_qp_offsets_present_flag << 7;
+	result.pps_info_flags |= pic->pps->weighted_pred_flag << 8;
+	result.pps_info_flags |= pic->pps->weighted_bipred_flag << 9;
+	result.pps_info_flags |= pic->pps->transquant_bypass_enabled_flag << 10;
+	result.pps_info_flags |= pic->pps->tiles_enabled_flag << 11;
+	result.pps_info_flags |= pic->pps->entropy_coding_sync_enabled_flag << 12;
+	result.pps_info_flags |= pic->pps->uniform_spacing_flag << 13;
+	result.pps_info_flags |= pic->pps->loop_filter_across_tiles_enabled_flag << 14;
+	result.pps_info_flags |= pic->pps->pps_loop_filter_across_slices_enabled_flag << 15;
+	result.pps_info_flags |= pic->pps->deblocking_filter_override_enabled_flag << 16;
+	result.pps_info_flags |= pic->pps->pps_deblocking_filter_disabled_flag << 17;
+	result.pps_info_flags |= pic->pps->lists_modification_present_flag << 18;
+	result.pps_info_flags |= pic->pps->slice_segment_header_extension_present_flag << 19;
+	//result.pps_info_flags |= pic->pps->deblocking_filter_control_present_flag; ???
+
+	result.num_extra_slice_header_bits = pic->pps->num_extra_slice_header_bits;
+	result.num_long_term_ref_pic_sps = pic->pps->sps->num_long_term_ref_pics_sps;
+	result.num_ref_idx_l0_default_active_minus1 = pic->pps->num_ref_idx_l0_default_active_minus1;
+	result.num_ref_idx_l1_default_active_minus1 = pic->pps->num_ref_idx_l1_default_active_minus1;
+	result.pps_cb_qp_offset = pic->pps->pps_cb_qp_offset;
+	result.pps_cr_qp_offset = pic->pps->pps_cr_qp_offset;
+	result.pps_beta_offset_div2 = pic->pps->pps_beta_offset_div2;
+	result.pps_tc_offset_div2 = pic->pps->pps_tc_offset_div2;
+	result.diff_cu_qp_delta_depth = pic->pps->diff_cu_qp_delta_depth;
+	result.num_tile_columns_minus1 = pic->pps->num_tile_columns_minus1;
+	result.num_tile_rows_minus1 = pic->pps->num_tile_rows_minus1;
+	result.log2_parallel_merge_level_minus2 = pic->pps->log2_parallel_merge_level_minus2;
+	result.init_qp_minus26 = pic->pps->init_qp_minus26;
+
+	for (i = 0; i < 19; ++i)
+		result.column_width_minus1[i] = pic->pps->column_width_minus1[i];
+
+	for (i = 0; i < 21; ++i)
+		result.row_height_minus1[i] = pic->pps->row_height_minus1[i];
+
+	result.num_delta_pocs_ref_rps_idx = pic->NumDeltaPocsOfRefRpsIdx;
+	result.curr_idx = pic->CurrPicOrderCntVal;
+	result.curr_poc = pic->CurrPicOrderCntVal;
+
+	vl_video_buffer_set_associated_data(target, &dec->base,
+					    (void *)(uintptr_t)pic->CurrPicOrderCntVal,
+					    &ruvd_destroy_associated_data);
+
+	for (i = 0; i < 16; ++i) {
+		struct pipe_video_buffer *ref = pic->ref[i];
+		uintptr_t ref_pic = 0;
+
+		result.poc_list[i] = pic->PicOrderCntVal[i];
+
+		if (ref)
+			ref_pic = (uintptr_t)vl_video_buffer_get_associated_data(ref, &dec->base);
+		else
+			ref_pic = 0x7F;
+		result.ref_pic_list[i] = ref_pic;
+	}
+
+	for (i = 0; i < 8; ++i) {
+		result.ref_pic_set_st_curr_before[i] = 0xFF;
+		result.ref_pic_set_st_curr_after[i] = 0xFF;
+		result.ref_pic_set_lt_curr[i] = 0xFF;
+	}
+
+	for (i = 0; i < pic->NumPocStCurrBefore; ++i)
+		result.ref_pic_set_st_curr_before[i] = pic->RefPicSetStCurrBefore[i];
+
+	for (i = 0; i < pic->NumPocStCurrAfter; ++i)
+		result.ref_pic_set_st_curr_after[i] = pic->RefPicSetStCurrAfter[i];
+
+	for (i = 0; i < pic->NumPocLtCurr; ++i)
+		result.ref_pic_set_lt_curr[i] = pic->RefPicSetLtCurr[i];
+
+	for (i = 0; i < 6; ++i)
+		result.ucScalingListDCCoefSizeID2[i] = pic->pps->sps->ScalingListDCCoeff16x16[i];
+
+	for (i = 0; i < 2; ++i)
+		result.ucScalingListDCCoefSizeID3[i] = pic->pps->sps->ScalingListDCCoeff32x32[i];
+
+	memcpy(dec->it, pic->pps->sps->ScalingList4x4, 6 * 16);
+	memcpy(dec->it + 96, pic->pps->sps->ScalingList8x8, 6 * 64);
+	memcpy(dec->it + 480, pic->pps->sps->ScalingList16x16, 6 * 64);
+	memcpy(dec->it + 864, pic->pps->sps->ScalingList32x32, 2 * 64);
+
+	for (i = 0 ; i < 2 ; i++) {
+		for (int j = 0 ; j < 15 ; j++)
+			result.direct_reflist[i][j] = pic->RefPicList[i][j];
+	}
+
+	if (pic->base.profile == PIPE_VIDEO_PROFILE_HEVC_MAIN_10) {
+		if (target->buffer_format == PIPE_FORMAT_P016) {
+			result.p010_mode = 1;
+			result.msb_mode = 1;
+		} else {
+			result.luma_10to8 = 5;
+			result.chroma_10to8 = 5;
+			result.sclr_luma10to8 = 4;
+			result.sclr_chroma10to8 = 4;
+		}
+	}
+
+	/* TODO
+	result.highestTid;
+	result.isNonRef;
+
+	IDRPicFlag;
+	RAPPicFlag;
+	NumPocTotalCurr;
+	NumShortTermPictureSliceHeaderBits;
+	NumLongTermPictureSliceHeaderBits;
+
+	IsLongTerm[16];
+	*/
+
+	return result;
+}
+
+/* get vc1 specific message bits */
+static struct ruvd_vc1 get_vc1_msg(struct pipe_vc1_picture_desc *pic)
+{
+	struct ruvd_vc1 result;
+
+	memset(&result, 0, sizeof(result));
+
+	switch(pic->base.profile) {
+	case PIPE_VIDEO_PROFILE_VC1_SIMPLE:
+		result.profile = RUVD_VC1_PROFILE_SIMPLE;
+		result.level = 1;
+		break;
+
+	case PIPE_VIDEO_PROFILE_VC1_MAIN:
+		result.profile = RUVD_VC1_PROFILE_MAIN;
+		result.level = 2;
+		break;
+
+	case PIPE_VIDEO_PROFILE_VC1_ADVANCED:
+		result.profile = RUVD_VC1_PROFILE_ADVANCED;
+		result.level = 4;
+		break;
+
+	default:
+		assert(0);
+	}
+
+	/* fields common for all profiles */
+	result.sps_info_flags |= pic->postprocflag << 7;
+	result.sps_info_flags |= pic->pulldown << 6;
+	result.sps_info_flags |= pic->interlace << 5;
+	result.sps_info_flags |= pic->tfcntrflag << 4;
+	result.sps_info_flags |= pic->finterpflag << 3;
+	result.sps_info_flags |= pic->psf << 1;
+
+	result.pps_info_flags |= pic->range_mapy_flag << 31;
+	result.pps_info_flags |= pic->range_mapy << 28;
+	result.pps_info_flags |= pic->range_mapuv_flag << 27;
+	result.pps_info_flags |= pic->range_mapuv << 24;
+	result.pps_info_flags |= pic->multires << 21;
+	result.pps_info_flags |= pic->maxbframes << 16;
+	result.pps_info_flags |= pic->overlap << 11;
+	result.pps_info_flags |= pic->quantizer << 9;
+	result.pps_info_flags |= pic->panscan_flag << 7;
+	result.pps_info_flags |= pic->refdist_flag << 6;
+	result.pps_info_flags |= pic->vstransform << 0;
+
+	/* some fields only apply to main/advanced profile */
+	if (pic->base.profile != PIPE_VIDEO_PROFILE_VC1_SIMPLE) {
+		result.pps_info_flags |= pic->syncmarker << 20;
+		result.pps_info_flags |= pic->rangered << 19;
+		result.pps_info_flags |= pic->loopfilter << 5;
+		result.pps_info_flags |= pic->fastuvmc << 4;
+		result.pps_info_flags |= pic->extended_mv << 3;
+		result.pps_info_flags |= pic->extended_dmv << 8;
+		result.pps_info_flags |= pic->dquant << 1;
+	}
+
+	result.chroma_format = 1;
+
+#if 0
+//(((unsigned int)(pPicParams->advance.reserved1))        << SPS_INFO_VC1_RESERVED_SHIFT)
+uint32_t 	slice_count
+uint8_t 	picture_type
+uint8_t 	frame_coding_mode
+uint8_t 	deblockEnable
+uint8_t 	pquant
+#endif
+
+	return result;
+}
+
+/* extract the frame number from a referenced video buffer */
+static uint32_t get_ref_pic_idx(struct ruvd_decoder *dec, struct pipe_video_buffer *ref)
+{
+	uint32_t min = MAX2(dec->frame_number, NUM_MPEG2_REFS) - NUM_MPEG2_REFS;
+	uint32_t max = MAX2(dec->frame_number, 1) - 1;
+	uintptr_t frame;
+
+	/* seems to be the most sane fallback */
+	if (!ref)
+		return max;
+
+	/* get the frame number from the associated data */
+	frame = (uintptr_t)vl_video_buffer_get_associated_data(ref, &dec->base);
+
+	/* limit the frame number to a valid range */
+	return MAX2(MIN2(frame, max), min);
+}
+
+/* get mpeg2 specific msg bits */
+static struct ruvd_mpeg2 get_mpeg2_msg(struct ruvd_decoder *dec,
+				       struct pipe_mpeg12_picture_desc *pic)
+{
+	const int *zscan = pic->alternate_scan ? vl_zscan_alternate : vl_zscan_normal;
+	struct ruvd_mpeg2 result;
+	unsigned i;
+
+	memset(&result, 0, sizeof(result));
+	result.decoded_pic_idx = dec->frame_number;
+	for (i = 0; i < 2; ++i)
+		result.ref_pic_idx[i] = get_ref_pic_idx(dec, pic->ref[i]);
+
+	result.load_intra_quantiser_matrix = 1;
+	result.load_nonintra_quantiser_matrix = 1;
+
+	for (i = 0; i < 64; ++i) {
+		result.intra_quantiser_matrix[i] = pic->intra_matrix[zscan[i]];
+		result.nonintra_quantiser_matrix[i] = pic->non_intra_matrix[zscan[i]];
+	}
+
+	result.profile_and_level_indication = 0;
+	result.chroma_format = 0x1;
+
+	result.picture_coding_type = pic->picture_coding_type;
+	result.f_code[0][0] = pic->f_code[0][0] + 1;
+	result.f_code[0][1] = pic->f_code[0][1] + 1;
+	result.f_code[1][0] = pic->f_code[1][0] + 1;
+	result.f_code[1][1] = pic->f_code[1][1] + 1;
+	result.intra_dc_precision = pic->intra_dc_precision;
+	result.pic_structure = pic->picture_structure;
+	result.top_field_first = pic->top_field_first;
+	result.frame_pred_frame_dct = pic->frame_pred_frame_dct;
+	result.concealment_motion_vectors = pic->concealment_motion_vectors;
+	result.q_scale_type = pic->q_scale_type;
+	result.intra_vlc_format = pic->intra_vlc_format;
+	result.alternate_scan = pic->alternate_scan;
+
+	return result;
+}
+
+/* get mpeg4 specific msg bits */
+static struct ruvd_mpeg4 get_mpeg4_msg(struct ruvd_decoder *dec,
+				       struct pipe_mpeg4_picture_desc *pic)
+{
+	struct ruvd_mpeg4 result;
+	unsigned i;
+
+	memset(&result, 0, sizeof(result));
+	result.decoded_pic_idx = dec->frame_number;
+	for (i = 0; i < 2; ++i)
+		result.ref_pic_idx[i] = get_ref_pic_idx(dec, pic->ref[i]);
+
+	result.variant_type = 0;
+	result.profile_and_level_indication = 0xF0; // ASP Level0
+
+	result.video_object_layer_verid = 0x5; // advanced simple
+	result.video_object_layer_shape = 0x0; // rectangular
+
+	result.video_object_layer_width = dec->base.width;
+	result.video_object_layer_height = dec->base.height;
+
+	result.vop_time_increment_resolution = pic->vop_time_increment_resolution;
+
+	result.flags |= pic->short_video_header << 0;
+	//result.flags |= obmc_disable << 1;
+	result.flags |= pic->interlaced << 2;
+        result.flags |= 1 << 3; // load_intra_quant_mat
+	result.flags |= 1 << 4; // load_nonintra_quant_mat
+	result.flags |= pic->quarter_sample << 5;
+	result.flags |= 1 << 6; // complexity_estimation_disable
+	result.flags |= pic->resync_marker_disable << 7;
+	//result.flags |= data_partitioned << 8;
+	//result.flags |= reversible_vlc << 9;
+	result.flags |= 0 << 10; // newpred_enable
+	result.flags |= 0 << 11; // reduced_resolution_vop_enable
+	//result.flags |= scalability << 12;
+	//result.flags |= is_object_layer_identifier << 13;
+	//result.flags |= fixed_vop_rate << 14;
+	//result.flags |= newpred_segment_type << 15;
+
+	result.quant_type = pic->quant_type;
+
+	for (i = 0; i < 64; ++i) {
+		result.intra_quant_mat[i] = pic->intra_matrix[vl_zscan_normal[i]];
+		result.nonintra_quant_mat[i] = pic->non_intra_matrix[vl_zscan_normal[i]];
+	}
+
+	/*
+	int32_t 	trd [2]
+	int32_t 	trb [2]
+	uint8_t 	vop_coding_type
+	uint8_t 	vop_fcode_forward
+	uint8_t 	vop_fcode_backward
+	uint8_t 	rounding_control
+	uint8_t 	alternate_vertical_scan_flag
+	uint8_t 	top_field_first
+	*/
+
+	return result;
+}
+
+static void get_mjpeg_slice_header(struct ruvd_decoder *dec, struct pipe_mjpeg_picture_desc *pic)
+{
+	int size = 0, saved_size, len_pos, i;
+	uint16_t *bs;
+	uint8_t *buf = dec->bs_ptr;
+
+	/* SOI */
+	buf[size++] = 0xff;
+	buf[size++] = 0xd8;
+
+	/* DQT */
+	buf[size++] = 0xff;
+	buf[size++] = 0xdb;
+
+	len_pos = size++;
+	size++;
+
+	for (i = 0; i < 4; ++i) {
+		if (pic->quantization_table.load_quantiser_table[i] == 0)
+			continue;
+
+		buf[size++] = i;
+		memcpy((buf + size), &pic->quantization_table.quantiser_table[i], 64);
+		size += 64;
+	}
+
+	bs = (uint16_t*)&buf[len_pos];
+	*bs = util_bswap16(size - 4);
+
+	saved_size = size;
+
+	/* DHT */
+	buf[size++] = 0xff;
+	buf[size++] = 0xc4;
+
+	len_pos = size++;
+	size++;
+
+	for (i = 0; i < 2; ++i) {
+		if (pic->huffman_table.load_huffman_table[i] == 0)
+			continue;
+
+		buf[size++] = 0x00 | i;
+		memcpy((buf + size), &pic->huffman_table.table[i].num_dc_codes, 16);
+		size += 16;
+		memcpy((buf + size), &pic->huffman_table.table[i].dc_values, 12);
+		size += 12;
+	}
+
+	for (i = 0; i < 2; ++i) {
+		if (pic->huffman_table.load_huffman_table[i] == 0)
+			continue;
+
+		buf[size++] = 0x10 | i;
+		memcpy((buf + size), &pic->huffman_table.table[i].num_ac_codes, 16);
+		size += 16;
+		memcpy((buf + size), &pic->huffman_table.table[i].ac_values, 162);
+		size += 162;
+	}
+
+	bs = (uint16_t*)&buf[len_pos];
+	*bs = util_bswap16(size - saved_size - 2);
+
+	saved_size = size;
+
+	/* DRI */
+	if (pic->slice_parameter.restart_interval) {
+		buf[size++] = 0xff;
+		buf[size++] = 0xdd;
+		buf[size++] = 0x00;
+		buf[size++] = 0x04;
+		bs = (uint16_t*)&buf[size++];
+		*bs = util_bswap16(pic->slice_parameter.restart_interval);
+		saved_size = ++size;
+	}
+
+	/* SOF */
+	buf[size++] = 0xff;
+	buf[size++] = 0xc0;
+
+	len_pos = size++;
+	size++;
+
+	buf[size++] = 0x08;
+
+	bs = (uint16_t*)&buf[size++];
+	*bs = util_bswap16(pic->picture_parameter.picture_height);
+	size++;
+
+	bs = (uint16_t*)&buf[size++];
+	*bs = util_bswap16(pic->picture_parameter.picture_width);
+	size++;
+
+	buf[size++] = pic->picture_parameter.num_components;
+
+	for (i = 0; i < pic->picture_parameter.num_components; ++i) {
+		buf[size++] = pic->picture_parameter.components[i].component_id;
+		buf[size++] = pic->picture_parameter.components[i].h_sampling_factor << 4 |
+			pic->picture_parameter.components[i].v_sampling_factor;
+		buf[size++] = pic->picture_parameter.components[i].quantiser_table_selector;
+	}
+
+	bs = (uint16_t*)&buf[len_pos];
+	*bs = util_bswap16(size - saved_size - 2);
+
+	saved_size = size;
+
+	/* SOS */
+	buf[size++] = 0xff;
+	buf[size++] = 0xda;
+
+	len_pos = size++;
+	size++;
+
+	buf[size++] = pic->slice_parameter.num_components;
+
+	for (i = 0; i < pic->slice_parameter.num_components; ++i) {
+		buf[size++] = pic->slice_parameter.components[i].component_selector;
+		buf[size++] = pic->slice_parameter.components[i].dc_table_selector << 4 |
+			pic->slice_parameter.components[i].ac_table_selector;
+	}
+
+	buf[size++] = 0x00;
+	buf[size++] = 0x3f;
+	buf[size++] = 0x00;
+
+	bs = (uint16_t*)&buf[len_pos];
+	*bs = util_bswap16(size - saved_size - 2);
+
+	dec->bs_ptr += size;
+	dec->bs_size += size;
+}
+
+/**
+ * destroy this video decoder
+ */
+static void ruvd_destroy(struct pipe_video_codec *decoder)
+{
+	struct ruvd_decoder *dec = (struct ruvd_decoder*)decoder;
+	unsigned i;
+
+	assert(decoder);
+
+	map_msg_fb_it_buf(dec);
+	dec->msg->size = sizeof(*dec->msg);
+	dec->msg->msg_type = RUVD_MSG_DESTROY;
+	dec->msg->stream_handle = dec->stream_handle;
+	send_msg_buf(dec);
+
+	flush(dec, 0);
+
+	dec->ws->cs_destroy(dec->cs);
+
+	for (i = 0; i < NUM_BUFFERS; ++i) {
+		rvid_destroy_buffer(&dec->msg_fb_it_buffers[i]);
+		rvid_destroy_buffer(&dec->bs_buffers[i]);
+	}
+
+	rvid_destroy_buffer(&dec->dpb);
+	rvid_destroy_buffer(&dec->ctx);
+	rvid_destroy_buffer(&dec->sessionctx);
+
+	FREE(dec);
+}
+
+/**
+ * start decoding of a new frame
+ */
+static void ruvd_begin_frame(struct pipe_video_codec *decoder,
+			     struct pipe_video_buffer *target,
+			     struct pipe_picture_desc *picture)
+{
+	struct ruvd_decoder *dec = (struct ruvd_decoder*)decoder;
+	uintptr_t frame;
+
+	assert(decoder);
+
+	frame = ++dec->frame_number;
+	vl_video_buffer_set_associated_data(target, decoder, (void *)frame,
+					    &ruvd_destroy_associated_data);
+
+	dec->bs_size = 0;
+	dec->bs_ptr = dec->ws->buffer_map(
+		dec->bs_buffers[dec->cur_buffer].res->buf,
+		dec->cs, PIPE_TRANSFER_WRITE);
+}
+
+/**
+ * decode a macroblock
+ */
+static void ruvd_decode_macroblock(struct pipe_video_codec *decoder,
+				   struct pipe_video_buffer *target,
+				   struct pipe_picture_desc *picture,
+				   const struct pipe_macroblock *macroblocks,
+				   unsigned num_macroblocks)
+{
+	/* not supported (yet) */
+	assert(0);
+}
+
+/**
+ * decode a bitstream
+ */
+static void ruvd_decode_bitstream(struct pipe_video_codec *decoder,
+				  struct pipe_video_buffer *target,
+				  struct pipe_picture_desc *picture,
+				  unsigned num_buffers,
+				  const void * const *buffers,
+				  const unsigned *sizes)
+{
+	struct ruvd_decoder *dec = (struct ruvd_decoder*)decoder;
+	enum pipe_video_format format = u_reduce_video_profile(picture->profile);
+	unsigned i;
+
+	assert(decoder);
+
+	if (!dec->bs_ptr)
+		return;
+
+	if (format == PIPE_VIDEO_FORMAT_JPEG)
+		get_mjpeg_slice_header(dec, (struct pipe_mjpeg_picture_desc*)picture);
+
+	for (i = 0; i < num_buffers; ++i) {
+		struct rvid_buffer *buf = &dec->bs_buffers[dec->cur_buffer];
+		unsigned new_size = dec->bs_size + sizes[i];
+
+		if (format == PIPE_VIDEO_FORMAT_JPEG)
+			new_size += 2; /* save for EOI */
+
+		if (new_size > buf->res->buf->size) {
+			dec->ws->buffer_unmap(buf->res->buf);
+			if (!rvid_resize_buffer(dec->screen, dec->cs, buf, new_size)) {
+				RVID_ERR("Can't resize bitstream buffer!");
+				return;
+			}
+
+			dec->bs_ptr = dec->ws->buffer_map(buf->res->buf, dec->cs,
+							  PIPE_TRANSFER_WRITE);
+			if (!dec->bs_ptr)
+				return;
+
+			dec->bs_ptr += dec->bs_size;
+		}
+
+		memcpy(dec->bs_ptr, buffers[i], sizes[i]);
+		dec->bs_size += sizes[i];
+		dec->bs_ptr += sizes[i];
+	}
+
+	if (format == PIPE_VIDEO_FORMAT_JPEG) {
+		((uint8_t *)dec->bs_ptr)[0] = 0xff;	/* EOI */
+		((uint8_t *)dec->bs_ptr)[1] = 0xd9;
+		dec->bs_size += 2;
+		dec->bs_ptr += 2;
+	}
+}
+
+/**
+ * end decoding of the current frame
+ */
+static void ruvd_end_frame(struct pipe_video_codec *decoder,
+			   struct pipe_video_buffer *target,
+			   struct pipe_picture_desc *picture)
+{
+	struct ruvd_decoder *dec = (struct ruvd_decoder*)decoder;
+	struct pb_buffer *dt;
+	struct rvid_buffer *msg_fb_it_buf, *bs_buf;
+	unsigned bs_size;
+
+	assert(decoder);
+
+	if (!dec->bs_ptr)
+		return;
+
+	msg_fb_it_buf = &dec->msg_fb_it_buffers[dec->cur_buffer];
+	bs_buf = &dec->bs_buffers[dec->cur_buffer];
+
+	bs_size = align(dec->bs_size, 128);
+	memset(dec->bs_ptr, 0, bs_size - dec->bs_size);
+	dec->ws->buffer_unmap(bs_buf->res->buf);
+
+	map_msg_fb_it_buf(dec);
+	dec->msg->size = sizeof(*dec->msg);
+	dec->msg->msg_type = RUVD_MSG_DECODE;
+	dec->msg->stream_handle = dec->stream_handle;
+	dec->msg->status_report_feedback_number = dec->frame_number;
+
+	dec->msg->body.decode.stream_type = dec->stream_type;
+	dec->msg->body.decode.decode_flags = 0x1;
+	dec->msg->body.decode.width_in_samples = dec->base.width;
+	dec->msg->body.decode.height_in_samples = dec->base.height;
+
+	if ((picture->profile == PIPE_VIDEO_PROFILE_VC1_SIMPLE) ||
+	    (picture->profile == PIPE_VIDEO_PROFILE_VC1_MAIN)) {
+		dec->msg->body.decode.width_in_samples = align(dec->msg->body.decode.width_in_samples, 16) / 16;
+		dec->msg->body.decode.height_in_samples = align(dec->msg->body.decode.height_in_samples, 16) / 16;
+	}
+
+	if (dec->dpb.res)
+		dec->msg->body.decode.dpb_size = dec->dpb.res->buf->size;
+	dec->msg->body.decode.bsd_size = bs_size;
+	dec->msg->body.decode.db_pitch = align(dec->base.width, get_db_pitch_alignment(dec));
+
+	if (dec->stream_type == RUVD_CODEC_H264_PERF &&
+	    ((struct r600_common_screen*)dec->screen)->family >= CHIP_POLARIS10)
+		dec->msg->body.decode.dpb_reserved = dec->ctx.res->buf->size;
+
+	dt = dec->set_dtb(dec->msg, (struct vl_video_buffer *)target);
+	if (((struct r600_common_screen*)dec->screen)->family >= CHIP_STONEY)
+		dec->msg->body.decode.dt_wa_chroma_top_offset = dec->msg->body.decode.dt_pitch / 2;
+
+	switch (u_reduce_video_profile(picture->profile)) {
+	case PIPE_VIDEO_FORMAT_MPEG4_AVC:
+		dec->msg->body.decode.codec.h264 = get_h264_msg(dec, (struct pipe_h264_picture_desc*)picture);
+		break;
+
+	case PIPE_VIDEO_FORMAT_HEVC:
+		dec->msg->body.decode.codec.h265 = get_h265_msg(dec, target, (struct pipe_h265_picture_desc*)picture);
+		if (dec->ctx.res == NULL) {
+			unsigned ctx_size;
+			if (dec->base.profile == PIPE_VIDEO_PROFILE_HEVC_MAIN_10)
+				ctx_size = calc_ctx_size_h265_main10(dec, (struct pipe_h265_picture_desc*)picture);
+			else
+				ctx_size = calc_ctx_size_h265_main(dec);
+			if (!rvid_create_buffer(dec->screen, &dec->ctx, ctx_size, PIPE_USAGE_DEFAULT)) {
+				RVID_ERR("Can't allocated context buffer.\n");
+			}
+			rvid_clear_buffer(decoder->context, &dec->ctx);
+		}
+
+		if (dec->ctx.res)
+			dec->msg->body.decode.dpb_reserved = dec->ctx.res->buf->size;
+		break;
+
+	case PIPE_VIDEO_FORMAT_VC1:
+		dec->msg->body.decode.codec.vc1 = get_vc1_msg((struct pipe_vc1_picture_desc*)picture);
+		break;
+
+	case PIPE_VIDEO_FORMAT_MPEG12:
+		dec->msg->body.decode.codec.mpeg2 = get_mpeg2_msg(dec, (struct pipe_mpeg12_picture_desc*)picture);
+		break;
+
+	case PIPE_VIDEO_FORMAT_MPEG4:
+		dec->msg->body.decode.codec.mpeg4 = get_mpeg4_msg(dec, (struct pipe_mpeg4_picture_desc*)picture);
+		break;
+
+	case PIPE_VIDEO_FORMAT_JPEG:
+		break;
+
+	default:
+		assert(0);
+		return;
+	}
+
+	dec->msg->body.decode.db_surf_tile_config = dec->msg->body.decode.dt_surf_tile_config;
+	dec->msg->body.decode.extension_support = 0x1;
+
+	/* set at least the feedback buffer size */
+	dec->fb[0] = dec->fb_size;
+
+	send_msg_buf(dec);
+
+	if (dec->dpb.res)
+		send_cmd(dec, RUVD_CMD_DPB_BUFFER, dec->dpb.res->buf, 0,
+			RADEON_USAGE_READWRITE, RADEON_DOMAIN_VRAM);
+
+	if (dec->ctx.res)
+		send_cmd(dec, RUVD_CMD_CONTEXT_BUFFER, dec->ctx.res->buf, 0,
+			RADEON_USAGE_READWRITE, RADEON_DOMAIN_VRAM);
+	send_cmd(dec, RUVD_CMD_BITSTREAM_BUFFER, bs_buf->res->buf,
+		 0, RADEON_USAGE_READ, RADEON_DOMAIN_GTT);
+	send_cmd(dec, RUVD_CMD_DECODING_TARGET_BUFFER, dt, 0,
+		 RADEON_USAGE_WRITE, RADEON_DOMAIN_VRAM);
+	send_cmd(dec, RUVD_CMD_FEEDBACK_BUFFER, msg_fb_it_buf->res->buf,
+		 FB_BUFFER_OFFSET, RADEON_USAGE_WRITE, RADEON_DOMAIN_GTT);
+	if (have_it(dec))
+		send_cmd(dec, RUVD_CMD_ITSCALING_TABLE_BUFFER, msg_fb_it_buf->res->buf,
+			 FB_BUFFER_OFFSET + dec->fb_size, RADEON_USAGE_READ, RADEON_DOMAIN_GTT);
+	set_reg(dec, dec->reg.cntl, 1);
+
+	flush(dec, RADEON_FLUSH_ASYNC);
+	next_buffer(dec);
+}
+
+/**
+ * flush any outstanding command buffers to the hardware
+ */
+static void ruvd_flush(struct pipe_video_codec *decoder)
+{
+}
+
+/**
+ * create and UVD decoder
+ */
+struct pipe_video_codec *ruvd_create_decoder(struct pipe_context *context,
+					     const struct pipe_video_codec *templ,
+					     ruvd_set_dtb set_dtb)
+{
+	struct radeon_winsys* ws = ((struct r600_common_context *)context)->ws;
+	struct r600_common_context *rctx = (struct r600_common_context*)context;
+	unsigned dpb_size;
+	unsigned width = templ->width, height = templ->height;
+	unsigned bs_buf_size;
+	struct radeon_info info;
+	struct ruvd_decoder *dec;
+	int r, i;
+
+	ws->query_info(ws, &info);
+
+	switch(u_reduce_video_profile(templ->profile)) {
+	case PIPE_VIDEO_FORMAT_MPEG12:
+		if (templ->entrypoint > PIPE_VIDEO_ENTRYPOINT_BITSTREAM || info.family < CHIP_PALM)
+			return vl_create_mpeg12_decoder(context, templ);
+
+		/* fall through */
+	case PIPE_VIDEO_FORMAT_MPEG4:
+		width = align(width, VL_MACROBLOCK_WIDTH);
+		height = align(height, VL_MACROBLOCK_HEIGHT);
+		break;
+	case PIPE_VIDEO_FORMAT_MPEG4_AVC:
+		width = align(width, VL_MACROBLOCK_WIDTH);
+		height = align(height, VL_MACROBLOCK_HEIGHT);
+		break;
+
+	default:
+		break;
+	}
+
+
+	dec = CALLOC_STRUCT(ruvd_decoder);
+
+	if (!dec)
+		return NULL;
+
+	if (info.drm_major < 3)
+		dec->use_legacy = true;
+
+	dec->base = *templ;
+	dec->base.context = context;
+	dec->base.width = width;
+	dec->base.height = height;
+
+	dec->base.destroy = ruvd_destroy;
+	dec->base.begin_frame = ruvd_begin_frame;
+	dec->base.decode_macroblock = ruvd_decode_macroblock;
+	dec->base.decode_bitstream = ruvd_decode_bitstream;
+	dec->base.end_frame = ruvd_end_frame;
+	dec->base.flush = ruvd_flush;
+
+	dec->stream_type = profile2stream_type(dec, info.family);
+	dec->set_dtb = set_dtb;
+	dec->stream_handle = rvid_alloc_stream_handle();
+	dec->screen = context->screen;
+	dec->ws = ws;
+	dec->cs = ws->cs_create(rctx->ctx, RING_UVD, NULL, NULL);
+	if (!dec->cs) {
+		RVID_ERR("Can't get command submission context.\n");
+		goto error;
+	}
+
+	dec->fb_size = (info.family == CHIP_TONGA) ? FB_BUFFER_SIZE_TONGA :
+			FB_BUFFER_SIZE;
+	bs_buf_size = width * height * (512 / (16 * 16));
+	for (i = 0; i < NUM_BUFFERS; ++i) {
+		unsigned msg_fb_it_size = FB_BUFFER_OFFSET + dec->fb_size;
+		STATIC_ASSERT(sizeof(struct ruvd_msg) <= FB_BUFFER_OFFSET);
+		if (have_it(dec))
+			msg_fb_it_size += IT_SCALING_TABLE_SIZE;
+		if (!rvid_create_buffer(dec->screen, &dec->msg_fb_it_buffers[i],
+					msg_fb_it_size, PIPE_USAGE_STAGING)) {
+			RVID_ERR("Can't allocated message buffers.\n");
+			goto error;
+		}
+
+		if (!rvid_create_buffer(dec->screen, &dec->bs_buffers[i],
+					bs_buf_size, PIPE_USAGE_STAGING)) {
+			RVID_ERR("Can't allocated bitstream buffers.\n");
+			goto error;
+		}
+
+		rvid_clear_buffer(context, &dec->msg_fb_it_buffers[i]);
+		rvid_clear_buffer(context, &dec->bs_buffers[i]);
+	}
+
+	dpb_size = calc_dpb_size(dec);
+	if (dpb_size) {
+		if (!rvid_create_buffer(dec->screen, &dec->dpb, dpb_size, PIPE_USAGE_DEFAULT)) {
+			RVID_ERR("Can't allocated dpb.\n");
+			goto error;
+		}
+		rvid_clear_buffer(context, &dec->dpb);
+	}
+
+	if (dec->stream_type == RUVD_CODEC_H264_PERF && info.family >= CHIP_POLARIS10) {
+		unsigned ctx_size = calc_ctx_size_h264_perf(dec);
+		if (!rvid_create_buffer(dec->screen, &dec->ctx, ctx_size, PIPE_USAGE_DEFAULT)) {
+			RVID_ERR("Can't allocated context buffer.\n");
+			goto error;
+		}
+		rvid_clear_buffer(context, &dec->ctx);
+	}
+
+	if (info.family >= CHIP_POLARIS10 && info.drm_minor >= 3) {
+		if (!rvid_create_buffer(dec->screen, &dec->sessionctx,
+					UVD_SESSION_CONTEXT_SIZE,
+					PIPE_USAGE_DEFAULT)) {
+			RVID_ERR("Can't allocated session ctx.\n");
+			goto error;
+		}
+		rvid_clear_buffer(context, &dec->sessionctx);
+	}
+
+	if (info.family >= CHIP_VEGA10) {
+		dec->reg.data0 = RUVD_GPCOM_VCPU_DATA0_SOC15;
+		dec->reg.data1 = RUVD_GPCOM_VCPU_DATA1_SOC15;
+		dec->reg.cmd = RUVD_GPCOM_VCPU_CMD_SOC15;
+		dec->reg.cntl = RUVD_ENGINE_CNTL_SOC15;
+	} else {
+		dec->reg.data0 = RUVD_GPCOM_VCPU_DATA0;
+		dec->reg.data1 = RUVD_GPCOM_VCPU_DATA1;
+		dec->reg.cmd = RUVD_GPCOM_VCPU_CMD;
+		dec->reg.cntl = RUVD_ENGINE_CNTL;
+	}
+
+	map_msg_fb_it_buf(dec);
+	dec->msg->size = sizeof(*dec->msg);
+	dec->msg->msg_type = RUVD_MSG_CREATE;
+	dec->msg->stream_handle = dec->stream_handle;
+	dec->msg->body.create.stream_type = dec->stream_type;
+	dec->msg->body.create.width_in_samples = dec->base.width;
+	dec->msg->body.create.height_in_samples = dec->base.height;
+	dec->msg->body.create.dpb_size = dpb_size;
+	send_msg_buf(dec);
+	r = flush(dec, 0);
+	if (r)
+		goto error;
+
+	next_buffer(dec);
+
+	return &dec->base;
+
+error:
+	if (dec->cs) dec->ws->cs_destroy(dec->cs);
+
+	for (i = 0; i < NUM_BUFFERS; ++i) {
+		rvid_destroy_buffer(&dec->msg_fb_it_buffers[i]);
+		rvid_destroy_buffer(&dec->bs_buffers[i]);
+	}
+
+	rvid_destroy_buffer(&dec->dpb);
+	rvid_destroy_buffer(&dec->ctx);
+	rvid_destroy_buffer(&dec->sessionctx);
+
+	FREE(dec);
+
+	return NULL;
+}
+
+/* calculate top/bottom offset */
+static unsigned texture_offset(struct radeon_surf *surface, unsigned layer,
+				enum ruvd_surface_type type)
+{
+	switch (type) {
+	default:
+	case RUVD_SURFACE_TYPE_LEGACY:
+		return surface->u.legacy.level[0].offset +
+			layer * surface->u.legacy.level[0].slice_size;
+		break;
+	case RUVD_SURFACE_TYPE_GFX9:
+		return surface->u.gfx9.surf_offset +
+			layer * surface->u.gfx9.surf_slice_size;
+		break;
+	}
+}
+
+/* hw encode the aspect of macro tiles */
+static unsigned macro_tile_aspect(unsigned macro_tile_aspect)
+{
+	switch (macro_tile_aspect) {
+	default:
+	case 1: macro_tile_aspect = 0;  break;
+	case 2: macro_tile_aspect = 1;  break;
+	case 4: macro_tile_aspect = 2;  break;
+	case 8: macro_tile_aspect = 3;  break;
+	}
+	return macro_tile_aspect;
+}
+
+/* hw encode the bank width and height */
+static unsigned bank_wh(unsigned bankwh)
+{
+	switch (bankwh) {
+	default:
+	case 1: bankwh = 0;     break;
+	case 2: bankwh = 1;     break;
+	case 4: bankwh = 2;     break;
+	case 8: bankwh = 3;     break;
+	}
+	return bankwh;
+}
+
+/**
+ * fill decoding target field from the luma and chroma surfaces
+ */
+void ruvd_set_dt_surfaces(struct ruvd_msg *msg, struct radeon_surf *luma,
+			struct radeon_surf *chroma, enum ruvd_surface_type type)
+{
+	switch (type) {
+	default:
+	case RUVD_SURFACE_TYPE_LEGACY:
+		msg->body.decode.dt_pitch = luma->u.legacy.level[0].nblk_x * luma->blk_w;
+		switch (luma->u.legacy.level[0].mode) {
+		case RADEON_SURF_MODE_LINEAR_ALIGNED:
+			msg->body.decode.dt_tiling_mode = RUVD_TILE_LINEAR;
+			msg->body.decode.dt_array_mode = RUVD_ARRAY_MODE_LINEAR;
+			break;
+		case RADEON_SURF_MODE_1D:
+			msg->body.decode.dt_tiling_mode = RUVD_TILE_8X8;
+			msg->body.decode.dt_array_mode = RUVD_ARRAY_MODE_1D_THIN;
+			break;
+		case RADEON_SURF_MODE_2D:
+			msg->body.decode.dt_tiling_mode = RUVD_TILE_8X8;
+			msg->body.decode.dt_array_mode = RUVD_ARRAY_MODE_2D_THIN;
+			break;
+		default:
+			assert(0);
+			break;
+		}
+
+		msg->body.decode.dt_luma_top_offset = texture_offset(luma, 0, type);
+		if (chroma)
+			msg->body.decode.dt_chroma_top_offset = texture_offset(chroma, 0, type);
+		if (msg->body.decode.dt_field_mode) {
+			msg->body.decode.dt_luma_bottom_offset = texture_offset(luma, 1, type);
+			if (chroma)
+				msg->body.decode.dt_chroma_bottom_offset = texture_offset(chroma, 1, type);
+		} else {
+			msg->body.decode.dt_luma_bottom_offset = msg->body.decode.dt_luma_top_offset;
+			msg->body.decode.dt_chroma_bottom_offset = msg->body.decode.dt_chroma_top_offset;
+		}
+
+		if (chroma) {
+			assert(luma->u.legacy.bankw == chroma->u.legacy.bankw);
+			assert(luma->u.legacy.bankh == chroma->u.legacy.bankh);
+			assert(luma->u.legacy.mtilea == chroma->u.legacy.mtilea);
+		}
+
+		msg->body.decode.dt_surf_tile_config |= RUVD_BANK_WIDTH(bank_wh(luma->u.legacy.bankw));
+		msg->body.decode.dt_surf_tile_config |= RUVD_BANK_HEIGHT(bank_wh(luma->u.legacy.bankh));
+		msg->body.decode.dt_surf_tile_config |= RUVD_MACRO_TILE_ASPECT_RATIO(macro_tile_aspect(luma->u.legacy.mtilea));
+		break;
+	case RUVD_SURFACE_TYPE_GFX9:
+		msg->body.decode.dt_pitch = luma->u.gfx9.surf_pitch * luma->blk_w;
+		/* SWIZZLE LINEAR MODE */
+		msg->body.decode.dt_tiling_mode = RUVD_TILE_LINEAR;
+		msg->body.decode.dt_array_mode = RUVD_ARRAY_MODE_LINEAR;
+		msg->body.decode.dt_luma_top_offset = texture_offset(luma, 0, type);
+		msg->body.decode.dt_chroma_top_offset = texture_offset(chroma, 0, type);
+		if (msg->body.decode.dt_field_mode) {
+			msg->body.decode.dt_luma_bottom_offset = texture_offset(luma, 1, type);
+			msg->body.decode.dt_chroma_bottom_offset = texture_offset(chroma, 1, type);
+		} else {
+			msg->body.decode.dt_luma_bottom_offset = msg->body.decode.dt_luma_top_offset;
+			msg->body.decode.dt_chroma_bottom_offset = msg->body.decode.dt_chroma_top_offset;
+		}
+		msg->body.decode.dt_surf_tile_config = 0;
+		break;
+	}
+}
diff --git a/src/gallium/drivers/r600/radeon_uvd.h b/src/gallium/drivers/r600/radeon_uvd.h
new file mode 100644
index 00000000000..a927c843dac
--- /dev/null
+++ b/src/gallium/drivers/r600/radeon_uvd.h
@@ -0,0 +1,447 @@
+/**************************************************************************
+ *
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+/*
+ * Authors:
+ *      Christian König <[email protected]>
+ *
+ */
+
+#ifndef RADEON_UVD_H
+#define RADEON_UVD_H
+
+#include "radeon/radeon_winsys.h"
+#include "vl/vl_video_buffer.h"
+
+/* UVD uses PM4 packet type 0 and 2 */
+#define RUVD_PKT_TYPE_S(x)		(((unsigned)(x) & 0x3) << 30)
+#define RUVD_PKT_TYPE_G(x)		(((x) >> 30) & 0x3)
+#define RUVD_PKT_TYPE_C			0x3FFFFFFF
+#define RUVD_PKT_COUNT_S(x)		(((unsigned)(x) & 0x3FFF) << 16)
+#define RUVD_PKT_COUNT_G(x)		(((x) >> 16) & 0x3FFF)
+#define RUVD_PKT_COUNT_C		0xC000FFFF
+#define RUVD_PKT0_BASE_INDEX_S(x)	(((unsigned)(x) & 0xFFFF) << 0)
+#define RUVD_PKT0_BASE_INDEX_G(x)	(((x) >> 0) & 0xFFFF)
+#define RUVD_PKT0_BASE_INDEX_C		0xFFFF0000
+#define RUVD_PKT0(index, count)		(RUVD_PKT_TYPE_S(0) | RUVD_PKT0_BASE_INDEX_S(index) | RUVD_PKT_COUNT_S(count))
+#define RUVD_PKT2()			(RUVD_PKT_TYPE_S(2))
+
+/* registers involved with UVD */
+#define RUVD_GPCOM_VCPU_CMD		0xEF0C
+#define RUVD_GPCOM_VCPU_DATA0		0xEF10
+#define RUVD_GPCOM_VCPU_DATA1		0xEF14
+#define RUVD_ENGINE_CNTL		0xEF18
+
+#define RUVD_GPCOM_VCPU_CMD_SOC15		0x2070c
+#define RUVD_GPCOM_VCPU_DATA0_SOC15		0x20710
+#define RUVD_GPCOM_VCPU_DATA1_SOC15		0x20714
+#define RUVD_ENGINE_CNTL_SOC15			0x20718
+
+/* UVD commands to VCPU */
+#define RUVD_CMD_MSG_BUFFER		0x00000000
+#define RUVD_CMD_DPB_BUFFER		0x00000001
+#define RUVD_CMD_DECODING_TARGET_BUFFER	0x00000002
+#define RUVD_CMD_FEEDBACK_BUFFER	0x00000003
+#define RUVD_CMD_SESSION_CONTEXT_BUFFER	0x00000005
+#define RUVD_CMD_BITSTREAM_BUFFER	0x00000100
+#define RUVD_CMD_ITSCALING_TABLE_BUFFER	0x00000204
+#define RUVD_CMD_CONTEXT_BUFFER		0x00000206
+
+/* UVD message types */
+#define RUVD_MSG_CREATE		0
+#define RUVD_MSG_DECODE		1
+#define RUVD_MSG_DESTROY	2
+
+/* UVD stream types */
+#define RUVD_CODEC_H264		0x00000000
+#define RUVD_CODEC_VC1		0x00000001
+#define RUVD_CODEC_MPEG2	0x00000003
+#define RUVD_CODEC_MPEG4	0x00000004
+#define RUVD_CODEC_H264_PERF	0x00000007
+#define RUVD_CODEC_MJPEG	0x00000008
+#define RUVD_CODEC_H265		0x00000010
+
+/* UVD decode target buffer tiling mode */
+#define RUVD_TILE_LINEAR	0x00000000
+#define RUVD_TILE_8X4		0x00000001
+#define RUVD_TILE_8X8		0x00000002
+#define RUVD_TILE_32AS8		0x00000003
+
+/* UVD decode target buffer array mode */
+#define RUVD_ARRAY_MODE_LINEAR				0x00000000
+#define RUVD_ARRAY_MODE_MACRO_LINEAR_MICRO_TILED	0x00000001
+#define RUVD_ARRAY_MODE_1D_THIN				0x00000002
+#define RUVD_ARRAY_MODE_2D_THIN				0x00000004
+#define RUVD_ARRAY_MODE_MACRO_TILED_MICRO_LINEAR	0x00000004
+#define RUVD_ARRAY_MODE_MACRO_TILED_MICRO_TILED		0x00000005
+
+/* UVD tile config */
+#define RUVD_BANK_WIDTH(x)		((x) << 0)
+#define RUVD_BANK_HEIGHT(x)		((x) << 3)
+#define RUVD_MACRO_TILE_ASPECT_RATIO(x)	((x) << 6)
+#define RUVD_NUM_BANKS(x)		((x) << 9)
+
+/* H.264 profile definitions */
+#define RUVD_H264_PROFILE_BASELINE	0x00000000
+#define RUVD_H264_PROFILE_MAIN		0x00000001
+#define RUVD_H264_PROFILE_HIGH		0x00000002
+#define RUVD_H264_PROFILE_STEREO_HIGH	0x00000003
+#define RUVD_H264_PROFILE_MVC		0x00000004
+
+/* VC-1 profile definitions */
+#define RUVD_VC1_PROFILE_SIMPLE		0x00000000
+#define RUVD_VC1_PROFILE_MAIN		0x00000001
+#define RUVD_VC1_PROFILE_ADVANCED	0x00000002
+
+enum ruvd_surface_type {
+	RUVD_SURFACE_TYPE_LEGACY = 0,
+	RUVD_SURFACE_TYPE_GFX9
+};
+
+struct ruvd_mvc_element {
+	uint16_t	viewOrderIndex;
+	uint16_t	viewId;
+	uint16_t	numOfAnchorRefsInL0;
+	uint16_t	viewIdOfAnchorRefsInL0[15];
+	uint16_t	numOfAnchorRefsInL1;
+	uint16_t	viewIdOfAnchorRefsInL1[15];
+	uint16_t	numOfNonAnchorRefsInL0;
+	uint16_t	viewIdOfNonAnchorRefsInL0[15];
+	uint16_t	numOfNonAnchorRefsInL1;
+	uint16_t	viewIdOfNonAnchorRefsInL1[15];
+};
+
+struct ruvd_h264 {
+	uint32_t	profile;
+	uint32_t	level;
+
+	uint32_t	sps_info_flags;
+	uint32_t	pps_info_flags;
+	uint8_t		chroma_format;
+	uint8_t		bit_depth_luma_minus8;
+	uint8_t		bit_depth_chroma_minus8;
+	uint8_t		log2_max_frame_num_minus4;
+
+	uint8_t		pic_order_cnt_type;
+	uint8_t		log2_max_pic_order_cnt_lsb_minus4;
+	uint8_t		num_ref_frames;
+	uint8_t		reserved_8bit;
+
+	int8_t		pic_init_qp_minus26;
+	int8_t		pic_init_qs_minus26;
+	int8_t		chroma_qp_index_offset;
+	int8_t		second_chroma_qp_index_offset;
+
+	uint8_t		num_slice_groups_minus1;
+	uint8_t		slice_group_map_type;
+	uint8_t		num_ref_idx_l0_active_minus1;
+	uint8_t		num_ref_idx_l1_active_minus1;
+
+	uint16_t	slice_group_change_rate_minus1;
+	uint16_t	reserved_16bit_1;
+
+	uint8_t		scaling_list_4x4[6][16];
+	uint8_t		scaling_list_8x8[2][64];
+
+	uint32_t	frame_num;
+	uint32_t	frame_num_list[16];
+	int32_t		curr_field_order_cnt_list[2];
+	int32_t		field_order_cnt_list[16][2];
+
+	uint32_t	decoded_pic_idx;
+
+	uint32_t	curr_pic_ref_frame_num;
+
+	uint8_t		ref_frame_list[16];
+
+	uint32_t	reserved[122];
+
+	struct {
+		uint32_t			numViews;
+		uint32_t			viewId0;
+		struct ruvd_mvc_element	mvcElements[1];
+	} mvc;
+};
+
+struct ruvd_h265 {
+	uint32_t	sps_info_flags;
+	uint32_t	pps_info_flags;
+
+	uint8_t		chroma_format;
+	uint8_t		bit_depth_luma_minus8;
+	uint8_t		bit_depth_chroma_minus8;
+	uint8_t		log2_max_pic_order_cnt_lsb_minus4;
+
+	uint8_t		sps_max_dec_pic_buffering_minus1;
+	uint8_t		log2_min_luma_coding_block_size_minus3;
+	uint8_t		log2_diff_max_min_luma_coding_block_size;
+	uint8_t		log2_min_transform_block_size_minus2;
+
+	uint8_t		log2_diff_max_min_transform_block_size;
+	uint8_t		max_transform_hierarchy_depth_inter;
+	uint8_t		max_transform_hierarchy_depth_intra;
+	uint8_t		pcm_sample_bit_depth_luma_minus1;
+
+	uint8_t		pcm_sample_bit_depth_chroma_minus1;
+	uint8_t		log2_min_pcm_luma_coding_block_size_minus3;
+	uint8_t		log2_diff_max_min_pcm_luma_coding_block_size;
+	uint8_t		num_extra_slice_header_bits;
+
+	uint8_t		num_short_term_ref_pic_sets;
+	uint8_t		num_long_term_ref_pic_sps;
+	uint8_t		num_ref_idx_l0_default_active_minus1;
+	uint8_t		num_ref_idx_l1_default_active_minus1;
+
+	int8_t		pps_cb_qp_offset;
+	int8_t		pps_cr_qp_offset;
+	int8_t		pps_beta_offset_div2;
+	int8_t		pps_tc_offset_div2;
+
+	uint8_t		diff_cu_qp_delta_depth;
+	uint8_t		num_tile_columns_minus1;
+	uint8_t		num_tile_rows_minus1;
+	uint8_t		log2_parallel_merge_level_minus2;
+
+	uint16_t	column_width_minus1[19];
+	uint16_t	row_height_minus1[21];
+
+	int8_t		init_qp_minus26;
+	uint8_t		num_delta_pocs_ref_rps_idx;
+	uint8_t		curr_idx;
+	uint8_t		reserved1;
+	int32_t		curr_poc;
+	uint8_t		ref_pic_list[16];
+	int32_t		poc_list[16];
+	uint8_t		ref_pic_set_st_curr_before[8];
+	uint8_t		ref_pic_set_st_curr_after[8];
+	uint8_t		ref_pic_set_lt_curr[8];
+
+	uint8_t		ucScalingListDCCoefSizeID2[6];
+	uint8_t		ucScalingListDCCoefSizeID3[2];
+
+	uint8_t		highestTid;
+	uint8_t		isNonRef;
+
+	uint8_t 	p010_mode;
+	uint8_t 	msb_mode;
+	uint8_t 	luma_10to8;
+	uint8_t 	chroma_10to8;
+	uint8_t 	sclr_luma10to8;
+	uint8_t 	sclr_chroma10to8;
+
+	uint8_t 	direct_reflist[2][15];
+};
+
+struct ruvd_vc1 {
+	uint32_t	profile;
+	uint32_t	level;
+	uint32_t	sps_info_flags;
+	uint32_t	pps_info_flags;
+	uint32_t	pic_structure;
+	uint32_t	chroma_format;
+};
+
+struct ruvd_mpeg2 {
+	uint32_t	decoded_pic_idx;
+	uint32_t	ref_pic_idx[2];
+
+	uint8_t		load_intra_quantiser_matrix;
+	uint8_t		load_nonintra_quantiser_matrix;
+	uint8_t		reserved_quantiser_alignement[2];
+	uint8_t		intra_quantiser_matrix[64];
+	uint8_t		nonintra_quantiser_matrix[64];
+
+	uint8_t		profile_and_level_indication;
+	uint8_t		chroma_format;
+
+	uint8_t		picture_coding_type;
+
+	uint8_t		reserved_1;
+
+	uint8_t		f_code[2][2];
+	uint8_t		intra_dc_precision;
+	uint8_t		pic_structure;
+	uint8_t		top_field_first;
+	uint8_t		frame_pred_frame_dct;
+	uint8_t		concealment_motion_vectors;
+	uint8_t		q_scale_type;
+	uint8_t		intra_vlc_format;
+	uint8_t		alternate_scan;
+};
+
+struct ruvd_mpeg4
+{
+	uint32_t	decoded_pic_idx;
+	uint32_t	ref_pic_idx[2];
+
+	uint32_t	variant_type;
+	uint8_t		profile_and_level_indication;
+
+	uint8_t		video_object_layer_verid;
+	uint8_t		video_object_layer_shape;
+
+	uint8_t		reserved_1;
+
+	uint16_t	video_object_layer_width;
+	uint16_t	video_object_layer_height;
+
+	uint16_t	vop_time_increment_resolution;
+
+	uint16_t	reserved_2;
+
+	uint32_t	flags;
+
+	uint8_t		quant_type;
+
+	uint8_t		reserved_3[3];
+
+	uint8_t		intra_quant_mat[64];
+	uint8_t		nonintra_quant_mat[64];
+
+	struct {
+		uint8_t		sprite_enable;
+
+		uint8_t		reserved_4[3];
+
+		uint16_t	sprite_width;
+		uint16_t	sprite_height;
+		int16_t		sprite_left_coordinate;
+		int16_t		sprite_top_coordinate;
+
+		uint8_t		no_of_sprite_warping_points;
+		uint8_t		sprite_warping_accuracy;
+		uint8_t		sprite_brightness_change;
+		uint8_t		low_latency_sprite_enable;
+	} sprite_config;
+
+	struct {
+		uint32_t	flags;
+		uint8_t		vol_mode;
+		uint8_t		reserved_5[3];
+	} divx_311_config;
+};
+
+/* message between driver and hardware */
+struct ruvd_msg {
+
+	uint32_t	size;
+	uint32_t	msg_type;
+	uint32_t	stream_handle;
+	uint32_t	status_report_feedback_number;
+
+	union {
+		struct {
+			uint32_t	stream_type;
+			uint32_t	session_flags;
+			uint32_t	asic_id;
+			uint32_t	width_in_samples;
+			uint32_t	height_in_samples;
+			uint32_t	dpb_buffer;
+			uint32_t	dpb_size;
+			uint32_t	dpb_model;
+			uint32_t	version_info;
+		} create;
+
+		struct {
+			uint32_t	stream_type;
+			uint32_t	decode_flags;
+			uint32_t	width_in_samples;
+			uint32_t	height_in_samples;
+
+			uint32_t	dpb_buffer;
+			uint32_t	dpb_size;
+			uint32_t	dpb_model;
+			uint32_t	dpb_reserved;
+
+			uint32_t	db_offset_alignment;
+			uint32_t	db_pitch;
+			uint32_t	db_tiling_mode;
+			uint32_t	db_array_mode;
+			uint32_t	db_field_mode;
+			uint32_t	db_surf_tile_config;
+			uint32_t	db_aligned_height;
+			uint32_t	db_reserved;
+
+			uint32_t	use_addr_macro;
+
+			uint32_t	bsd_buffer;
+			uint32_t	bsd_size;
+
+			uint32_t	pic_param_buffer;
+			uint32_t	pic_param_size;
+			uint32_t	mb_cntl_buffer;
+			uint32_t	mb_cntl_size;
+
+			uint32_t	dt_buffer;
+			uint32_t	dt_pitch;
+			uint32_t	dt_tiling_mode;
+			uint32_t	dt_array_mode;
+			uint32_t	dt_field_mode;
+			uint32_t	dt_luma_top_offset;
+			uint32_t	dt_luma_bottom_offset;
+			uint32_t	dt_chroma_top_offset;
+			uint32_t	dt_chroma_bottom_offset;
+			uint32_t	dt_surf_tile_config;
+			uint32_t	dt_uv_surf_tile_config;
+			// re-use dt_wa_chroma_top_offset as dt_ext_info for UV pitch in stoney
+			uint32_t	dt_wa_chroma_top_offset;
+			uint32_t	dt_wa_chroma_bottom_offset;
+
+			uint32_t	reserved[16];
+
+			union {
+				struct ruvd_h264	h264;
+				struct ruvd_h265	h265;
+				struct ruvd_vc1		vc1;
+				struct ruvd_mpeg2	mpeg2;
+				struct ruvd_mpeg4	mpeg4;
+
+				uint32_t info[768];
+			} codec;
+
+			uint8_t		extension_support;
+			uint8_t		reserved_8bit_1;
+			uint8_t		reserved_8bit_2;
+			uint8_t		reserved_8bit_3;
+			uint32_t	extension_reserved[64];
+		} decode;
+	} body;
+};
+
+/* driver dependent callback */
+typedef struct pb_buffer* (*ruvd_set_dtb)
+(struct ruvd_msg* msg, struct vl_video_buffer *vb);
+
+/* create an UVD decode */
+struct pipe_video_codec *ruvd_create_decoder(struct pipe_context *context,
+					     const struct pipe_video_codec *templat,
+					     ruvd_set_dtb set_dtb);
+
+/* fill decoding target field from the luma and chroma surfaces */
+void ruvd_set_dt_surfaces(struct ruvd_msg *msg, struct radeon_surf *luma,
+			struct radeon_surf *chroma, enum ruvd_surface_type type);
+#endif
diff --git a/src/gallium/drivers/r600/radeon_vce.c b/src/gallium/drivers/r600/radeon_vce.c
new file mode 100644
index 00000000000..e8b1a632dd8
--- /dev/null
+++ b/src/gallium/drivers/r600/radeon_vce.c
@@ -0,0 +1,553 @@
+/**************************************************************************
+ *
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+/*
+ * Authors:
+ *      Christian König <[email protected]>
+ *
+ */
+
+#include <stdio.h>
+
+#include "pipe/p_video_codec.h"
+
+#include "util/u_video.h"
+#include "util/u_memory.h"
+
+#include "vl/vl_video_buffer.h"
+
+#include "r600_pipe_common.h"
+#include "radeon_video.h"
+#include "radeon_vce.h"
+
+#define FW_40_2_2 ((40 << 24) | (2 << 16) | (2 << 8))
+#define FW_50_0_1 ((50 << 24) | (0 << 16) | (1 << 8))
+#define FW_50_1_2 ((50 << 24) | (1 << 16) | (2 << 8))
+#define FW_50_10_2 ((50 << 24) | (10 << 16) | (2 << 8))
+#define FW_50_17_3 ((50 << 24) | (17 << 16) | (3 << 8))
+#define FW_52_0_3 ((52 << 24) | (0 << 16) | (3 << 8))
+#define FW_52_4_3 ((52 << 24) | (4 << 16) | (3 << 8))
+#define FW_52_8_3 ((52 << 24) | (8 << 16) | (3 << 8))
+#define FW_53 (53 << 24)
+
+/**
+ * flush commands to the hardware
+ */
+static void flush(struct rvce_encoder *enc)
+{
+	enc->ws->cs_flush(enc->cs, RADEON_FLUSH_ASYNC, NULL);
+	enc->task_info_idx = 0;
+	enc->bs_idx = 0;
+}
+
+#if 0
+static void dump_feedback(struct rvce_encoder *enc, struct rvid_buffer *fb)
+{
+	uint32_t *ptr = enc->ws->buffer_map(fb->res->buf, enc->cs, PIPE_TRANSFER_READ_WRITE);
+	unsigned i = 0;
+	fprintf(stderr, "\n");
+	fprintf(stderr, "encStatus:\t\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "encHasBitstream:\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "encHasAudioBitstream:\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "encBitstreamOffset:\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "encBitstreamSize:\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "encAudioBitstreamOffset:\t%08x\n", ptr[i++]);
+	fprintf(stderr, "encAudioBitstreamSize:\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "encExtrabytes:\t\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "encAudioExtrabytes:\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "videoTimeStamp:\t\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "audioTimeStamp:\t\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "videoOutputType:\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "attributeFlags:\t\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "seiPrivatePackageOffset:\t%08x\n", ptr[i++]);
+	fprintf(stderr, "seiPrivatePackageSize:\t\t%08x\n", ptr[i++]);
+	fprintf(stderr, "\n");
+	enc->ws->buffer_unmap(fb->res->buf);
+}
+#endif
+
+/**
+ * reset the CPB handling
+ */
+static void reset_cpb(struct rvce_encoder *enc)
+{
+	unsigned i;
+
+	LIST_INITHEAD(&enc->cpb_slots);
+	for (i = 0; i < enc->cpb_num; ++i) {
+		struct rvce_cpb_slot *slot = &enc->cpb_array[i];
+		slot->index = i;
+		slot->picture_type = PIPE_H264_ENC_PICTURE_TYPE_SKIP;
+		slot->frame_num = 0;
+		slot->pic_order_cnt = 0;
+		LIST_ADDTAIL(&slot->list, &enc->cpb_slots);
+	}
+}
+
+/**
+ * sort l0 and l1 to the top of the list
+ */
+static void sort_cpb(struct rvce_encoder *enc)
+{
+	struct rvce_cpb_slot *i, *l0 = NULL, *l1 = NULL;
+
+	LIST_FOR_EACH_ENTRY(i, &enc->cpb_slots, list) {
+		if (i->frame_num == enc->pic.ref_idx_l0)
+			l0 = i;
+
+		if (i->frame_num == enc->pic.ref_idx_l1)
+			l1 = i;
+
+		if (enc->pic.picture_type == PIPE_H264_ENC_PICTURE_TYPE_P && l0)
+			break;
+
+		if (enc->pic.picture_type == PIPE_H264_ENC_PICTURE_TYPE_B &&
+		    l0 && l1)
+			break;
+	}
+
+	if (l1) {
+		LIST_DEL(&l1->list);
+		LIST_ADD(&l1->list, &enc->cpb_slots);
+	}
+
+	if (l0) {
+		LIST_DEL(&l0->list);
+		LIST_ADD(&l0->list, &enc->cpb_slots);
+	}
+}
+
+/**
+ * get number of cpbs based on dpb
+ */
+static unsigned get_cpb_num(struct rvce_encoder *enc)
+{
+	unsigned w = align(enc->base.width, 16) / 16;
+	unsigned h = align(enc->base.height, 16) / 16;
+	unsigned dpb;
+
+	switch (enc->base.level) {
+	case 10:
+		dpb = 396;
+		break;
+	case 11:
+		dpb = 900;
+		break;
+	case 12:
+	case 13:
+	case 20:
+		dpb = 2376;
+		break;
+	case 21:
+		dpb = 4752;
+		break;
+	case 22:
+	case 30:
+		dpb = 8100;
+		break;
+	case 31:
+		dpb = 18000;
+		break;
+	case 32:
+		dpb = 20480;
+		break;
+	case 40:
+	case 41:
+		dpb = 32768;
+		break;
+	case 42:
+		dpb = 34816;
+		break;
+	case 50:
+		dpb = 110400;
+		break;
+	default:
+	case 51:
+	case 52:
+		dpb = 184320;
+		break;
+	}
+
+	return MIN2(dpb / (w * h), 16);
+}
+
+/**
+ * Get the slot for the currently encoded frame
+ */
+struct rvce_cpb_slot *current_slot(struct rvce_encoder *enc)
+{
+	return LIST_ENTRY(struct rvce_cpb_slot, enc->cpb_slots.prev, list);
+}
+
+/**
+ * Get the slot for L0
+ */
+struct rvce_cpb_slot *l0_slot(struct rvce_encoder *enc)
+{
+	return LIST_ENTRY(struct rvce_cpb_slot, enc->cpb_slots.next, list);
+}
+
+/**
+ * Get the slot for L1
+ */
+struct rvce_cpb_slot *l1_slot(struct rvce_encoder *enc)
+{
+	return LIST_ENTRY(struct rvce_cpb_slot, enc->cpb_slots.next->next, list);
+}
+
+/**
+ * Calculate the offsets into the CPB
+ */
+void rvce_frame_offset(struct rvce_encoder *enc, struct rvce_cpb_slot *slot,
+		       signed *luma_offset, signed *chroma_offset)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen *)enc->screen;
+	unsigned pitch, vpitch, fsize;
+
+	if (rscreen->chip_class < GFX9) {
+		pitch = align(enc->luma->u.legacy.level[0].nblk_x * enc->luma->bpe, 128);
+		vpitch = align(enc->luma->u.legacy.level[0].nblk_y, 16);
+	} else {
+		pitch = align(enc->luma->u.gfx9.surf_pitch * enc->luma->bpe, 256);
+		vpitch = align(enc->luma->u.gfx9.surf_height, 16);
+	}
+	fsize = pitch * (vpitch + vpitch / 2);
+
+	*luma_offset = slot->index * fsize;
+	*chroma_offset = *luma_offset + pitch * vpitch;
+}
+
+/**
+ * destroy this video encoder
+ */
+static void rvce_destroy(struct pipe_video_codec *encoder)
+{
+	struct rvce_encoder *enc = (struct rvce_encoder*)encoder;
+	if (enc->stream_handle) {
+		struct rvid_buffer fb;
+		rvid_create_buffer(enc->screen, &fb, 512, PIPE_USAGE_STAGING);
+		enc->fb = &fb;
+		enc->session(enc);
+		enc->feedback(enc);
+		enc->destroy(enc);
+		flush(enc);
+		rvid_destroy_buffer(&fb);
+	}
+	rvid_destroy_buffer(&enc->cpb);
+	enc->ws->cs_destroy(enc->cs);
+	FREE(enc->cpb_array);
+	FREE(enc);
+}
+
+static void rvce_begin_frame(struct pipe_video_codec *encoder,
+			     struct pipe_video_buffer *source,
+			     struct pipe_picture_desc *picture)
+{
+	struct rvce_encoder *enc = (struct rvce_encoder*)encoder;
+	struct vl_video_buffer *vid_buf = (struct vl_video_buffer *)source;
+	struct pipe_h264_enc_picture_desc *pic = (struct pipe_h264_enc_picture_desc *)picture;
+
+	bool need_rate_control =
+		enc->pic.rate_ctrl.rate_ctrl_method != pic->rate_ctrl.rate_ctrl_method ||
+		enc->pic.quant_i_frames != pic->quant_i_frames ||
+		enc->pic.quant_p_frames != pic->quant_p_frames ||
+		enc->pic.quant_b_frames != pic->quant_b_frames;
+
+	enc->pic = *pic;
+	get_pic_param(enc, pic);
+
+	enc->get_buffer(vid_buf->resources[0], &enc->handle, &enc->luma);
+	enc->get_buffer(vid_buf->resources[1], NULL, &enc->chroma);
+
+	if (pic->picture_type == PIPE_H264_ENC_PICTURE_TYPE_IDR)
+		reset_cpb(enc);
+	else if (pic->picture_type == PIPE_H264_ENC_PICTURE_TYPE_P ||
+	         pic->picture_type == PIPE_H264_ENC_PICTURE_TYPE_B)
+		sort_cpb(enc);
+	
+	if (!enc->stream_handle) {
+		struct rvid_buffer fb;
+		enc->stream_handle = rvid_alloc_stream_handle();
+		rvid_create_buffer(enc->screen, &fb, 512, PIPE_USAGE_STAGING);
+		enc->fb = &fb;
+		enc->session(enc);
+		enc->create(enc);
+		enc->config(enc);
+		enc->feedback(enc);
+		flush(enc);
+		//dump_feedback(enc, &fb);
+		rvid_destroy_buffer(&fb);
+		need_rate_control = false;
+	}
+
+	if (need_rate_control) {
+		enc->session(enc);
+		enc->config(enc);
+		flush(enc);
+	}
+}
+
+static void rvce_encode_bitstream(struct pipe_video_codec *encoder,
+				  struct pipe_video_buffer *source,
+				  struct pipe_resource *destination,
+				  void **fb)
+{
+	struct rvce_encoder *enc = (struct rvce_encoder*)encoder;
+	enc->get_buffer(destination, &enc->bs_handle, NULL);
+	enc->bs_size = destination->width0;
+
+	*fb = enc->fb = CALLOC_STRUCT(rvid_buffer);
+	if (!rvid_create_buffer(enc->screen, enc->fb, 512, PIPE_USAGE_STAGING)) {
+		RVID_ERR("Can't create feedback buffer.\n");
+		return;
+	}
+	if (!radeon_emitted(enc->cs, 0))
+		enc->session(enc);
+	enc->encode(enc);
+	enc->feedback(enc);
+}
+
+static void rvce_end_frame(struct pipe_video_codec *encoder,
+			   struct pipe_video_buffer *source,
+			   struct pipe_picture_desc *picture)
+{
+	struct rvce_encoder *enc = (struct rvce_encoder*)encoder;
+	struct rvce_cpb_slot *slot = LIST_ENTRY(
+		struct rvce_cpb_slot, enc->cpb_slots.prev, list);
+
+	if (!enc->dual_inst || enc->bs_idx > 1)
+		flush(enc);
+
+	/* update the CPB backtrack with the just encoded frame */
+	slot->picture_type = enc->pic.picture_type;
+	slot->frame_num = enc->pic.frame_num;
+	slot->pic_order_cnt = enc->pic.pic_order_cnt;
+	if (!enc->pic.not_referenced) {
+		LIST_DEL(&slot->list);
+		LIST_ADD(&slot->list, &enc->cpb_slots);
+	}
+}
+
+static void rvce_get_feedback(struct pipe_video_codec *encoder,
+			      void *feedback, unsigned *size)
+{
+	struct rvce_encoder *enc = (struct rvce_encoder*)encoder;
+	struct rvid_buffer *fb = feedback;
+
+	if (size) {
+		uint32_t *ptr = enc->ws->buffer_map(fb->res->buf, enc->cs, PIPE_TRANSFER_READ_WRITE);
+
+		if (ptr[1]) {
+			*size = ptr[4] - ptr[9];
+		} else {
+			*size = 0;
+		}
+
+		enc->ws->buffer_unmap(fb->res->buf);
+	}
+	//dump_feedback(enc, fb);
+	rvid_destroy_buffer(fb);
+	FREE(fb);
+}
+
+/**
+ * flush any outstanding command buffers to the hardware
+ */
+static void rvce_flush(struct pipe_video_codec *encoder)
+{
+	struct rvce_encoder *enc = (struct rvce_encoder*)encoder;
+
+	flush(enc);
+}
+
+static void rvce_cs_flush(void *ctx, unsigned flags,
+			  struct pipe_fence_handle **fence)
+{
+	// just ignored
+}
+
+struct pipe_video_codec *rvce_create_encoder(struct pipe_context *context,
+					     const struct pipe_video_codec *templ,
+					     struct radeon_winsys* ws,
+					     rvce_get_buffer get_buffer)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen *)context->screen;
+	struct r600_common_context *rctx = (struct r600_common_context*)context;
+	struct rvce_encoder *enc;
+	struct pipe_video_buffer *tmp_buf, templat = {};
+	struct radeon_surf *tmp_surf;
+	unsigned cpb_size;
+
+	if (!rscreen->info.vce_fw_version) {
+		RVID_ERR("Kernel doesn't supports VCE!\n");
+		return NULL;
+
+	} else if (!rvce_is_fw_version_supported(rscreen)) {
+		RVID_ERR("Unsupported VCE fw version loaded!\n");
+		return NULL;
+	}
+
+	enc = CALLOC_STRUCT(rvce_encoder);
+	if (!enc)
+		return NULL;
+
+	if (rscreen->info.drm_major == 3)
+		enc->use_vm = true;
+	if ((rscreen->info.drm_major == 2 && rscreen->info.drm_minor >= 42) ||
+            rscreen->info.drm_major == 3)
+		enc->use_vui = true;
+	if (rscreen->info.family >= CHIP_TONGA &&
+	    rscreen->info.family != CHIP_STONEY &&
+	    rscreen->info.family != CHIP_POLARIS11 &&
+	    rscreen->info.family != CHIP_POLARIS12)
+		enc->dual_pipe = true;
+	/* TODO enable B frame with dual instance */
+	if ((rscreen->info.family >= CHIP_TONGA) &&
+		(templ->max_references == 1) &&
+		(rscreen->info.vce_harvest_config == 0))
+		enc->dual_inst = true;
+
+	enc->base = *templ;
+	enc->base.context = context;
+
+	enc->base.destroy = rvce_destroy;
+	enc->base.begin_frame = rvce_begin_frame;
+	enc->base.encode_bitstream = rvce_encode_bitstream;
+	enc->base.end_frame = rvce_end_frame;
+	enc->base.flush = rvce_flush;
+	enc->base.get_feedback = rvce_get_feedback;
+	enc->get_buffer = get_buffer;
+
+	enc->screen = context->screen;
+	enc->ws = ws;
+	enc->cs = ws->cs_create(rctx->ctx, RING_VCE, rvce_cs_flush, enc);
+	if (!enc->cs) {
+		RVID_ERR("Can't get command submission context.\n");
+		goto error;
+	}
+
+	templat.buffer_format = PIPE_FORMAT_NV12;
+	templat.chroma_format = PIPE_VIDEO_CHROMA_FORMAT_420;
+	templat.width = enc->base.width;
+	templat.height = enc->base.height;
+	templat.interlaced = false;
+	if (!(tmp_buf = context->create_video_buffer(context, &templat))) {
+		RVID_ERR("Can't create video buffer.\n");
+		goto error;
+	}
+
+	enc->cpb_num = get_cpb_num(enc);
+	if (!enc->cpb_num)
+		goto error;
+
+	get_buffer(((struct vl_video_buffer *)tmp_buf)->resources[0], NULL, &tmp_surf);
+
+	cpb_size = (rscreen->chip_class < GFX9) ?
+		align(tmp_surf->u.legacy.level[0].nblk_x * tmp_surf->bpe, 128) *
+		align(tmp_surf->u.legacy.level[0].nblk_y, 32) :
+
+		align(tmp_surf->u.gfx9.surf_pitch * tmp_surf->bpe, 256) *
+		align(tmp_surf->u.gfx9.surf_height, 32);
+
+	cpb_size = cpb_size * 3 / 2;
+	cpb_size = cpb_size * enc->cpb_num;
+	if (enc->dual_pipe)
+		cpb_size +=  RVCE_MAX_AUX_BUFFER_NUM *
+			RVCE_MAX_BITSTREAM_OUTPUT_ROW_SIZE * 2;
+	tmp_buf->destroy(tmp_buf);
+	if (!rvid_create_buffer(enc->screen, &enc->cpb, cpb_size, PIPE_USAGE_DEFAULT)) {
+		RVID_ERR("Can't create CPB buffer.\n");
+		goto error;
+	}
+
+	enc->cpb_array = CALLOC(enc->cpb_num, sizeof(struct rvce_cpb_slot));
+	if (!enc->cpb_array)
+		goto error;
+
+	reset_cpb(enc);
+
+	goto error;
+
+	return &enc->base;
+
+error:
+	if (enc->cs)
+		enc->ws->cs_destroy(enc->cs);
+
+	rvid_destroy_buffer(&enc->cpb);
+
+	FREE(enc->cpb_array);
+	FREE(enc);
+	return NULL;
+}
+
+/**
+ * check if kernel has the right fw version loaded
+ */
+bool rvce_is_fw_version_supported(struct r600_common_screen *rscreen)
+{
+	switch (rscreen->info.vce_fw_version) {
+	case FW_40_2_2:
+	case FW_50_0_1:
+	case FW_50_1_2:
+	case FW_50_10_2:
+	case FW_50_17_3:
+	case FW_52_0_3:
+	case FW_52_4_3:
+	case FW_52_8_3:
+		return true;
+	default:
+		if ((rscreen->info.vce_fw_version & (0xff << 24)) == FW_53)
+			return true;
+		else
+			return false;
+	}
+}
+
+/**
+ * Add the buffer as relocation to the current command submission
+ */
+void rvce_add_buffer(struct rvce_encoder *enc, struct pb_buffer *buf,
+                     enum radeon_bo_usage usage, enum radeon_bo_domain domain,
+                     signed offset)
+{
+	int reloc_idx;
+
+	reloc_idx = enc->ws->cs_add_buffer(enc->cs, buf, usage | RADEON_USAGE_SYNCHRONIZED,
+					   domain, RADEON_PRIO_VCE);
+	if (enc->use_vm) {
+		uint64_t addr;
+		addr = enc->ws->buffer_get_virtual_address(buf);
+		addr = addr + offset;
+		RVCE_CS(addr >> 32);
+		RVCE_CS(addr);
+	} else {
+		offset += enc->ws->buffer_get_reloc_offset(buf);
+		RVCE_CS(reloc_idx * 4);
+		RVCE_CS(offset);
+	}
+}
diff --git a/src/gallium/drivers/r600/radeon_vce.h b/src/gallium/drivers/r600/radeon_vce.h
new file mode 100644
index 00000000000..f79e65c9ac2
--- /dev/null
+++ b/src/gallium/drivers/r600/radeon_vce.h
@@ -0,0 +1,462 @@
+/**************************************************************************
+ *
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+/*
+ * Authors:
+ *      Christian König <[email protected]>
+ *
+ */
+
+#ifndef RADEON_VCE_H
+#define RADEON_VCE_H
+
+#include "util/list.h"
+
+#define RVCE_CS(value) (enc->cs->current.buf[enc->cs->current.cdw++] = (value))
+#define RVCE_BEGIN(cmd) { \
+	uint32_t *begin = &enc->cs->current.buf[enc->cs->current.cdw++]; \
+	RVCE_CS(cmd)
+#define RVCE_READ(buf, domain, off) rvce_add_buffer(enc, (buf), RADEON_USAGE_READ, (domain), (off))
+#define RVCE_WRITE(buf, domain, off) rvce_add_buffer(enc, (buf), RADEON_USAGE_WRITE, (domain), (off))
+#define RVCE_READWRITE(buf, domain, off) rvce_add_buffer(enc, (buf), RADEON_USAGE_READWRITE, (domain), (off))
+#define RVCE_END() *begin = (&enc->cs->current.buf[enc->cs->current.cdw] - begin) * 4; }
+
+#define RVCE_MAX_BITSTREAM_OUTPUT_ROW_SIZE (4096 * 16 * 2.5)
+#define RVCE_MAX_AUX_BUFFER_NUM 4
+
+struct r600_common_screen;
+
+/* driver dependent callback */
+typedef void (*rvce_get_buffer)(struct pipe_resource *resource,
+				struct pb_buffer **handle,
+				struct radeon_surf **surface);
+
+/* Coded picture buffer slot */
+struct rvce_cpb_slot {
+	struct list_head		list;
+
+	unsigned			index;
+	enum pipe_h264_enc_picture_type	picture_type;
+	unsigned			frame_num;
+	unsigned			pic_order_cnt;
+};
+
+struct rvce_rate_control {
+	uint32_t		rc_method;
+	uint32_t		target_bitrate;
+	uint32_t		peak_bitrate;
+	uint32_t		frame_rate_num;
+	uint32_t		gop_size;
+	uint32_t		quant_i_frames;
+	uint32_t		quant_p_frames;
+	uint32_t		quant_b_frames;
+	uint32_t		vbv_buffer_size;
+	uint32_t		frame_rate_den;
+	uint32_t		vbv_buf_lv;
+	uint32_t		max_au_size;
+	uint32_t		qp_initial_mode;
+	uint32_t		target_bits_picture;
+	uint32_t		peak_bits_picture_integer;
+	uint32_t		peak_bits_picture_fraction;
+	uint32_t		min_qp;
+	uint32_t		max_qp;
+	uint32_t		skip_frame_enable;
+	uint32_t		fill_data_enable;
+	uint32_t		enforce_hrd;
+	uint32_t		b_pics_delta_qp;
+	uint32_t		ref_b_pics_delta_qp;
+	uint32_t		rc_reinit_disable;
+	uint32_t		enc_lcvbr_init_qp_flag;
+	uint32_t		lcvbrsatd_based_nonlinear_bit_budget_flag;
+};
+
+struct rvce_motion_estimation {
+	uint32_t		enc_ime_decimation_search;
+	uint32_t		motion_est_half_pixel;
+	uint32_t		motion_est_quarter_pixel;
+	uint32_t		disable_favor_pmv_point;
+	uint32_t		force_zero_point_center;
+	uint32_t		lsmvert;
+	uint32_t		enc_search_range_x;
+	uint32_t		enc_search_range_y;
+	uint32_t		enc_search1_range_x;
+	uint32_t		enc_search1_range_y;
+	uint32_t		disable_16x16_frame1;
+	uint32_t		disable_satd;
+	uint32_t		enable_amd;
+	uint32_t		enc_disable_sub_mode;
+	uint32_t		enc_ime_skip_x;
+	uint32_t		enc_ime_skip_y;
+	uint32_t		enc_en_ime_overw_dis_subm;
+	uint32_t		enc_ime_overw_dis_subm_no;
+	uint32_t		enc_ime2_search_range_x;
+	uint32_t		enc_ime2_search_range_y;
+	uint32_t		parallel_mode_speedup_enable;
+	uint32_t		fme0_enc_disable_sub_mode;
+	uint32_t		fme1_enc_disable_sub_mode;
+	uint32_t		ime_sw_speedup_enable;
+};
+
+struct rvce_pic_control {
+	uint32_t		enc_use_constrained_intra_pred;
+	uint32_t		enc_cabac_enable;
+	uint32_t		enc_cabac_idc;
+	uint32_t		enc_loop_filter_disable;
+	int32_t			enc_lf_beta_offset;
+	int32_t			enc_lf_alpha_c0_offset;
+	uint32_t		enc_crop_left_offset;
+	uint32_t		enc_crop_right_offset;
+	uint32_t		enc_crop_top_offset;
+	uint32_t		enc_crop_bottom_offset;
+	uint32_t		enc_num_mbs_per_slice;
+	uint32_t		enc_intra_refresh_num_mbs_per_slot;
+	uint32_t		enc_force_intra_refresh;
+	uint32_t		enc_force_imb_period;
+	uint32_t		enc_pic_order_cnt_type;
+	uint32_t		log2_max_pic_order_cnt_lsb_minus4;
+	uint32_t		enc_sps_id;
+	uint32_t		enc_pps_id;
+	uint32_t		enc_constraint_set_flags;
+	uint32_t		enc_b_pic_pattern;
+	uint32_t		weight_pred_mode_b_picture;
+	uint32_t		enc_number_of_reference_frames;
+	uint32_t		enc_max_num_ref_frames;
+	uint32_t		enc_num_default_active_ref_l0;
+	uint32_t		enc_num_default_active_ref_l1;
+	uint32_t		enc_slice_mode;
+	uint32_t		enc_max_slice_size;
+};
+
+struct rvce_task_info {
+	uint32_t		offset_of_next_task_info;
+	uint32_t		task_operation;
+	uint32_t		reference_picture_dependency;
+	uint32_t		collocate_flag_dependency;
+	uint32_t		feedback_index;
+	uint32_t		video_bitstream_ring_index;
+};
+
+struct rvce_feedback_buf_pkg {
+	uint32_t		feedback_ring_address_hi;
+	uint32_t		feedback_ring_address_lo;
+	uint32_t		feedback_ring_size;
+};
+
+struct rvce_rdo {
+	uint32_t		enc_disable_tbe_pred_i_frame;
+	uint32_t		enc_disable_tbe_pred_p_frame;
+	uint32_t		use_fme_interpol_y;
+	uint32_t		use_fme_interpol_uv;
+	uint32_t		use_fme_intrapol_y;
+	uint32_t		use_fme_intrapol_uv;
+	uint32_t		use_fme_interpol_y_1;
+	uint32_t		use_fme_interpol_uv_1;
+	uint32_t		use_fme_intrapol_y_1;
+	uint32_t		use_fme_intrapol_uv_1;
+	uint32_t		enc_16x16_cost_adj;
+	uint32_t		enc_skip_cost_adj;
+	uint32_t		enc_force_16x16_skip;
+	uint32_t		enc_disable_threshold_calc_a;
+	uint32_t		enc_luma_coeff_cost;
+	uint32_t		enc_luma_mb_coeff_cost;
+	uint32_t		enc_chroma_coeff_cost;
+};
+
+struct rvce_vui {
+	uint32_t		aspect_ratio_info_present_flag;
+	uint32_t		aspect_ratio_idc;
+	uint32_t		sar_width;
+	uint32_t		sar_height;
+	uint32_t		overscan_info_present_flag;
+	uint32_t		overscan_Approp_flag;
+	uint32_t		video_signal_type_present_flag;
+	uint32_t		video_format;
+	uint32_t		video_full_range_flag;
+	uint32_t		color_description_present_flag;
+	uint32_t		color_prim;
+	uint32_t		transfer_char;
+	uint32_t		matrix_coef;
+	uint32_t		chroma_loc_info_present_flag;
+	uint32_t		chroma_loc_top;
+	uint32_t		chroma_loc_bottom;
+	uint32_t		timing_info_present_flag;
+	uint32_t		num_units_in_tick;
+	uint32_t		time_scale;
+	uint32_t		fixed_frame_rate_flag;
+	uint32_t		nal_hrd_parameters_present_flag;
+	uint32_t		cpb_cnt_minus1;
+	uint32_t		bit_rate_scale;
+	uint32_t		cpb_size_scale;
+	uint32_t		bit_rate_value_minus;
+	uint32_t		cpb_size_value_minus;
+	uint32_t		cbr_flag;
+	uint32_t		initial_cpb_removal_delay_length_minus1;
+	uint32_t		cpb_removal_delay_length_minus1;
+	uint32_t		dpb_output_delay_length_minus1;
+	uint32_t		time_offset_length;
+	uint32_t		low_delay_hrd_flag;
+	uint32_t		pic_struct_present_flag;
+	uint32_t		bitstream_restriction_present_flag;
+	uint32_t		motion_vectors_over_pic_boundaries_flag;
+	uint32_t		max_bytes_per_pic_denom;
+	uint32_t		max_bits_per_mb_denom;
+	uint32_t		log2_max_mv_length_hori;
+	uint32_t		log2_max_mv_length_vert;
+	uint32_t		num_reorder_frames;
+	uint32_t		max_dec_frame_buffering;
+};
+
+struct rvce_enc_operation {
+	uint32_t		insert_headers;
+	uint32_t		picture_structure;
+	uint32_t		allowed_max_bitstream_size;
+	uint32_t		force_refresh_map;
+	uint32_t		insert_aud;
+	uint32_t		end_of_sequence;
+	uint32_t		end_of_stream;
+	uint32_t		input_picture_luma_address_hi;
+	uint32_t		input_picture_luma_address_lo;
+	uint32_t		input_picture_chroma_address_hi;
+	uint32_t		input_picture_chroma_address_lo;
+	uint32_t		enc_input_frame_y_pitch;
+	uint32_t		enc_input_pic_luma_pitch;
+	uint32_t		enc_input_pic_chroma_pitch;;
+	uint32_t		enc_input_pic_addr_array;
+	uint32_t		enc_input_pic_addr_array_disable2pipe_disablemboffload;
+	uint32_t		enc_input_pic_tile_config;
+	uint32_t		enc_pic_type;
+	uint32_t		enc_idr_flag;
+	uint32_t		enc_idr_pic_id;
+	uint32_t		enc_mgs_key_pic;
+	uint32_t		enc_reference_flag;
+	uint32_t		enc_temporal_layer_index;
+	uint32_t		num_ref_idx_active_override_flag;
+	uint32_t		num_ref_idx_l0_active_minus1;
+	uint32_t		num_ref_idx_l1_active_minus1;
+	uint32_t		enc_ref_list_modification_op;
+	uint32_t		enc_ref_list_modification_num;
+	uint32_t		enc_decoded_picture_marking_op;
+	uint32_t		enc_decoded_picture_marking_num;
+	uint32_t		enc_decoded_picture_marking_idx;
+	uint32_t		enc_decoded_ref_base_picture_marking_op;
+	uint32_t		enc_decoded_ref_base_picture_marking_num;
+	uint32_t		l0_picture_structure;
+	uint32_t		l0_enc_pic_type;
+	uint32_t		l0_frame_number;
+	uint32_t		l0_picture_order_count;
+	uint32_t		l0_luma_offset;
+	uint32_t		l0_chroma_offset;
+	uint32_t		l1_picture_structure;
+	uint32_t		l1_enc_pic_type;
+	uint32_t		l1_frame_number;
+	uint32_t		l1_picture_order_count;
+	uint32_t		l1_luma_offset;
+	uint32_t		l1_chroma_offset;
+	uint32_t		enc_reconstructed_luma_offset;
+	uint32_t		enc_reconstructed_chroma_offset;;
+	uint32_t		enc_coloc_buffer_offset;
+	uint32_t		enc_reconstructed_ref_base_picture_luma_offset;
+	uint32_t		enc_reconstructed_ref_base_picture_chroma_offset;
+	uint32_t		enc_reference_ref_base_picture_luma_offset;
+	uint32_t		enc_reference_ref_base_picture_chroma_offset;
+	uint32_t		picture_count;
+	uint32_t		frame_number;
+	uint32_t		picture_order_count;
+	uint32_t		num_i_pic_remain_in_rcgop;
+	uint32_t		num_p_pic_remain_in_rcgop;
+	uint32_t		num_b_pic_remain_in_rcgop;
+	uint32_t		num_ir_pic_remain_in_rcgop;
+	uint32_t		enable_intra_refresh;
+	uint32_t		aq_variance_en;
+	uint32_t		aq_block_size;
+	uint32_t		aq_mb_variance_sel;
+	uint32_t		aq_frame_variance_sel;
+	uint32_t		aq_param_a;
+	uint32_t		aq_param_b;
+	uint32_t		aq_param_c;
+	uint32_t		aq_param_d;
+	uint32_t		aq_param_e;
+	uint32_t		context_in_sfb;
+};
+
+struct rvce_enc_create {
+	uint32_t		enc_use_circular_buffer;
+	uint32_t		enc_profile;
+	uint32_t		enc_level;
+	uint32_t		enc_pic_struct_restriction;
+	uint32_t		enc_image_width;
+	uint32_t		enc_image_height;
+	uint32_t		enc_ref_pic_luma_pitch;
+	uint32_t		enc_ref_pic_chroma_pitch;
+	uint32_t		enc_ref_y_height_in_qw;
+	uint32_t		enc_ref_pic_addr_array_enc_pic_struct_restriction_disable_rdo;
+	uint32_t		enc_pre_encode_context_buffer_offset;
+	uint32_t		enc_pre_encode_input_luma_buffer_offset;
+	uint32_t		enc_pre_encode_input_chroma_buffer_offset;
+	uint32_t		enc_pre_encode_mode_chromaflag_vbaqmode_scenechangesensitivity;
+};
+
+struct rvce_config_ext {
+	uint32_t		enc_enable_perf_logging;
+};
+
+struct rvce_h264_enc_pic {
+	struct rvce_rate_control rc;
+	struct rvce_motion_estimation me;
+	struct rvce_pic_control pc;
+	struct rvce_task_info ti;
+	struct rvce_feedback_buf_pkg fb;
+	struct rvce_rdo rdo;
+	struct rvce_vui vui;
+	struct rvce_enc_operation eo;
+	struct rvce_enc_create ec;
+	struct rvce_config_ext ce;
+
+	unsigned quant_i_frames;
+	unsigned quant_p_frames;
+	unsigned quant_b_frames;
+
+	enum pipe_h264_enc_picture_type picture_type;
+	unsigned frame_num;
+	unsigned frame_num_cnt;
+	unsigned p_remain;
+	unsigned i_remain;
+	unsigned idr_pic_id;
+	unsigned gop_cnt;
+	unsigned gop_size;
+	unsigned pic_order_cnt;
+	unsigned ref_idx_l0;
+	unsigned ref_idx_l1;
+	unsigned addrmode_arraymode_disrdo_distwoinstants;
+
+	bool not_referenced;
+	bool is_idr;
+	bool has_ref_pic_list;
+	bool enable_vui;
+	unsigned int ref_pic_list_0[32];
+	unsigned int ref_pic_list_1[32];
+	unsigned int frame_idx[32];
+};
+
+/* VCE encoder representation */
+struct rvce_encoder {
+	struct pipe_video_codec		base;
+
+	/* version specific packets */
+	void (*session)(struct rvce_encoder *enc);
+	void (*create)(struct rvce_encoder *enc);
+	void (*feedback)(struct rvce_encoder *enc);
+	void (*rate_control)(struct rvce_encoder *enc);
+	void (*config_extension)(struct rvce_encoder *enc);
+	void (*pic_control)(struct rvce_encoder *enc);
+	void (*motion_estimation)(struct rvce_encoder *enc);
+	void (*rdo)(struct rvce_encoder *enc);
+	void (*vui)(struct rvce_encoder *enc);
+	void (*config)(struct rvce_encoder *enc);
+	void (*encode)(struct rvce_encoder *enc);
+	void (*destroy)(struct rvce_encoder *enc);
+	void (*task_info)(struct rvce_encoder *enc, uint32_t op,
+			  uint32_t dep, uint32_t fb_idx,
+			  uint32_t ring_idx);
+
+	unsigned			stream_handle;
+
+	struct pipe_screen		*screen;
+	struct radeon_winsys*		ws;
+	struct radeon_winsys_cs*	cs;
+
+	rvce_get_buffer			get_buffer;
+
+	struct pb_buffer*	handle;
+	struct radeon_surf*		luma;
+	struct radeon_surf*		chroma;
+
+	struct pb_buffer*	bs_handle;
+	unsigned			bs_size;
+
+	struct rvce_cpb_slot		*cpb_array;
+	struct list_head		cpb_slots;
+	unsigned			cpb_num;
+
+	struct rvid_buffer		*fb;
+	struct rvid_buffer		cpb;
+	struct pipe_h264_enc_picture_desc pic;
+	struct rvce_h264_enc_pic	enc_pic;
+
+	unsigned			task_info_idx;
+	unsigned			bs_idx;
+
+	bool				use_vm;
+	bool				use_vui;
+	bool				dual_pipe;
+	bool				dual_inst;
+};
+
+/* CPB handling functions */
+struct rvce_cpb_slot *current_slot(struct rvce_encoder *enc);
+struct rvce_cpb_slot *l0_slot(struct rvce_encoder *enc);
+struct rvce_cpb_slot *l1_slot(struct rvce_encoder *enc);
+void rvce_frame_offset(struct rvce_encoder *enc, struct rvce_cpb_slot *slot,
+		       signed *luma_offset, signed *chroma_offset);
+
+struct pipe_video_codec *rvce_create_encoder(struct pipe_context *context,
+					     const struct pipe_video_codec *templat,
+					     struct radeon_winsys* ws,
+					     rvce_get_buffer get_buffer);
+
+bool rvce_is_fw_version_supported(struct r600_common_screen *rscreen);
+
+void rvce_add_buffer(struct rvce_encoder *enc, struct pb_buffer *buf,
+		     enum radeon_bo_usage usage, enum radeon_bo_domain domain,
+		     signed offset);
+
+/* init vce fw 40.2.2 specific callbacks */
+void radeon_vce_40_2_2_init(struct rvce_encoder *enc);
+
+/* init vce fw 50 specific callbacks */
+void radeon_vce_50_init(struct rvce_encoder *enc);
+
+/* init vce fw 52 specific callbacks */
+void radeon_vce_52_init(struct rvce_encoder *enc);
+
+/* version specific function for getting parameters */
+void (*get_pic_param)(struct rvce_encoder *enc,
+                      struct pipe_h264_enc_picture_desc *pic);
+
+/* get parameters for vce 40.2.2 */
+void radeon_vce_40_2_2_get_param(struct rvce_encoder *enc,
+                                 struct pipe_h264_enc_picture_desc *pic);
+
+/* get parameters for vce 50 */
+void radeon_vce_50_get_param(struct rvce_encoder *enc,
+                             struct pipe_h264_enc_picture_desc *pic);
+
+/* get parameters for vce 52 */
+void radeon_vce_52_get_param(struct rvce_encoder *enc,
+                             struct pipe_h264_enc_picture_desc *pic);
+
+#endif
diff --git a/src/gallium/drivers/r600/radeon_video.c b/src/gallium/drivers/r600/radeon_video.c
new file mode 100644
index 00000000000..99b6676fee1
--- /dev/null
+++ b/src/gallium/drivers/r600/radeon_video.c
@@ -0,0 +1,372 @@
+/**************************************************************************
+ *
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+/*
+ * Authors:
+ *      Christian König <[email protected]>
+ *
+ */
+
+#include <unistd.h>
+
+#include "util/u_memory.h"
+#include "util/u_video.h"
+
+#include "vl/vl_defines.h"
+#include "vl/vl_video_buffer.h"
+
+#include "r600_pipe_common.h"
+#include "radeon_video.h"
+#include "radeon_vce.h"
+
+#define UVD_FW_1_66_16 ((1 << 24) | (66 << 16) | (16 << 8))
+
+/* generate an stream handle */
+unsigned rvid_alloc_stream_handle()
+{
+	static unsigned counter = 0;
+	unsigned stream_handle = 0;
+	unsigned pid = getpid();
+	int i;
+
+	for (i = 0; i < 32; ++i)
+		stream_handle |= ((pid >> i) & 1) << (31 - i);
+
+	stream_handle ^= ++counter;
+	return stream_handle;
+}
+
+/* create a buffer in the winsys */
+bool rvid_create_buffer(struct pipe_screen *screen, struct rvid_buffer *buffer,
+			unsigned size, unsigned usage)
+{
+	memset(buffer, 0, sizeof(*buffer));
+	buffer->usage = usage;
+
+	/* Hardware buffer placement restrictions require the kernel to be
+	 * able to move buffers around individually, so request a
+	 * non-sub-allocated buffer.
+	 */
+	buffer->res = (struct r600_resource *)
+		pipe_buffer_create(screen, PIPE_BIND_SHARED,
+				   usage, size);
+
+	return buffer->res != NULL;
+}
+
+/* destroy a buffer */
+void rvid_destroy_buffer(struct rvid_buffer *buffer)
+{
+	r600_resource_reference(&buffer->res, NULL);
+}
+
+/* reallocate a buffer, preserving its content */
+bool rvid_resize_buffer(struct pipe_screen *screen, struct radeon_winsys_cs *cs,
+			struct rvid_buffer *new_buf, unsigned new_size)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen *)screen;
+	struct radeon_winsys* ws = rscreen->ws;
+	unsigned bytes = MIN2(new_buf->res->buf->size, new_size);
+	struct rvid_buffer old_buf = *new_buf;
+	void *src = NULL, *dst = NULL;
+
+	if (!rvid_create_buffer(screen, new_buf, new_size, new_buf->usage))
+		goto error;
+
+	src = ws->buffer_map(old_buf.res->buf, cs, PIPE_TRANSFER_READ);
+	if (!src)
+		goto error;
+
+	dst = ws->buffer_map(new_buf->res->buf, cs, PIPE_TRANSFER_WRITE);
+	if (!dst)
+		goto error;
+
+	memcpy(dst, src, bytes);
+	if (new_size > bytes) {
+		new_size -= bytes;
+		dst += bytes;
+		memset(dst, 0, new_size);
+	}
+	ws->buffer_unmap(new_buf->res->buf);
+	ws->buffer_unmap(old_buf.res->buf);
+	rvid_destroy_buffer(&old_buf);
+	return true;
+
+error:
+	if (src)
+		ws->buffer_unmap(old_buf.res->buf);
+	rvid_destroy_buffer(new_buf);
+	*new_buf = old_buf;
+	return false;
+}
+
+/* clear the buffer with zeros */
+void rvid_clear_buffer(struct pipe_context *context, struct rvid_buffer* buffer)
+{
+	struct r600_common_context *rctx = (struct r600_common_context*)context;
+
+	rctx->dma_clear_buffer(context, &buffer->res->b.b, 0,
+			       buffer->res->buf->size, 0);
+	context->flush(context, NULL, 0);
+}
+
+/**
+ * join surfaces into the same buffer with identical tiling params
+ * sumup their sizes and replace the backend buffers with a single bo
+ */
+void rvid_join_surfaces(struct r600_common_context *rctx,
+			struct pb_buffer** buffers[VL_NUM_COMPONENTS],
+			struct radeon_surf *surfaces[VL_NUM_COMPONENTS])
+{
+	struct radeon_winsys* ws;
+	unsigned best_tiling, best_wh, off;
+	unsigned size, alignment;
+	struct pb_buffer *pb;
+	unsigned i, j;
+
+	ws = rctx->ws;
+
+	for (i = 0, best_tiling = 0, best_wh = ~0; i < VL_NUM_COMPONENTS; ++i) {
+		unsigned wh;
+
+		if (!surfaces[i])
+			continue;
+
+		if (rctx->chip_class < GFX9) {
+			/* choose the smallest bank w/h for now */
+			wh = surfaces[i]->u.legacy.bankw * surfaces[i]->u.legacy.bankh;
+			if (wh < best_wh) {
+				best_wh = wh;
+				best_tiling = i;
+			}
+		}
+	}
+
+	for (i = 0, off = 0; i < VL_NUM_COMPONENTS; ++i) {
+		if (!surfaces[i])
+			continue;
+
+		/* adjust the texture layer offsets */
+		off = align(off, surfaces[i]->surf_alignment);
+
+		if (rctx->chip_class < GFX9) {
+			/* copy the tiling parameters */
+			surfaces[i]->u.legacy.bankw = surfaces[best_tiling]->u.legacy.bankw;
+			surfaces[i]->u.legacy.bankh = surfaces[best_tiling]->u.legacy.bankh;
+			surfaces[i]->u.legacy.mtilea = surfaces[best_tiling]->u.legacy.mtilea;
+			surfaces[i]->u.legacy.tile_split = surfaces[best_tiling]->u.legacy.tile_split;
+
+			for (j = 0; j < ARRAY_SIZE(surfaces[i]->u.legacy.level); ++j)
+				surfaces[i]->u.legacy.level[j].offset += off;
+		} else
+			surfaces[i]->u.gfx9.surf_offset += off;
+
+		off += surfaces[i]->surf_size;
+	}
+
+	for (i = 0, size = 0, alignment = 0; i < VL_NUM_COMPONENTS; ++i) {
+		if (!buffers[i] || !*buffers[i])
+			continue;
+
+		size = align(size, (*buffers[i])->alignment);
+		size += (*buffers[i])->size;
+		alignment = MAX2(alignment, (*buffers[i])->alignment * 1);
+	}
+
+	if (!size)
+		return;
+
+	/* TODO: 2D tiling workaround */
+	alignment *= 2;
+
+	pb = ws->buffer_create(ws, size, alignment, RADEON_DOMAIN_VRAM,
+			       RADEON_FLAG_GTT_WC);
+	if (!pb)
+		return;
+
+	for (i = 0; i < VL_NUM_COMPONENTS; ++i) {
+		if (!buffers[i] || !*buffers[i])
+			continue;
+
+		pb_reference(buffers[i], pb);
+	}
+
+	pb_reference(&pb, NULL);
+}
+
+int rvid_get_video_param(struct pipe_screen *screen,
+			 enum pipe_video_profile profile,
+			 enum pipe_video_entrypoint entrypoint,
+			 enum pipe_video_cap param)
+{
+	struct r600_common_screen *rscreen = (struct r600_common_screen *)screen;
+	enum pipe_video_format codec = u_reduce_video_profile(profile);
+	struct radeon_info info;
+
+	rscreen->ws->query_info(rscreen->ws, &info);
+
+	if (entrypoint == PIPE_VIDEO_ENTRYPOINT_ENCODE) {
+		switch (param) {
+		case PIPE_VIDEO_CAP_SUPPORTED:
+			return codec == PIPE_VIDEO_FORMAT_MPEG4_AVC &&
+				rvce_is_fw_version_supported(rscreen);
+		case PIPE_VIDEO_CAP_NPOT_TEXTURES:
+			return 1;
+		case PIPE_VIDEO_CAP_MAX_WIDTH:
+			return (rscreen->family < CHIP_TONGA) ? 2048 : 4096;
+		case PIPE_VIDEO_CAP_MAX_HEIGHT:
+			return (rscreen->family < CHIP_TONGA) ? 1152 : 2304;
+		case PIPE_VIDEO_CAP_PREFERED_FORMAT:
+			return PIPE_FORMAT_NV12;
+		case PIPE_VIDEO_CAP_PREFERS_INTERLACED:
+			return false;
+		case PIPE_VIDEO_CAP_SUPPORTS_INTERLACED:
+			return false;
+		case PIPE_VIDEO_CAP_SUPPORTS_PROGRESSIVE:
+			return true;
+		case PIPE_VIDEO_CAP_STACKED_FRAMES:
+			return (rscreen->family < CHIP_TONGA) ? 1 : 2;
+		default:
+			return 0;
+		}
+	}
+
+	switch (param) {
+	case PIPE_VIDEO_CAP_SUPPORTED:
+		switch (codec) {
+		case PIPE_VIDEO_FORMAT_MPEG12:
+			return profile != PIPE_VIDEO_PROFILE_MPEG1;
+		case PIPE_VIDEO_FORMAT_MPEG4:
+			/* no support for MPEG4 on older hw */
+			return rscreen->family >= CHIP_PALM;
+		case PIPE_VIDEO_FORMAT_MPEG4_AVC:
+			if ((rscreen->family == CHIP_POLARIS10 ||
+			     rscreen->family == CHIP_POLARIS11) &&
+			    info.uvd_fw_version < UVD_FW_1_66_16 ) {
+				RVID_ERR("POLARIS10/11 firmware version need to be updated.\n");
+				return false;
+			}
+			return true;
+		case PIPE_VIDEO_FORMAT_VC1:
+			return true;
+		case PIPE_VIDEO_FORMAT_HEVC:
+			/* Carrizo only supports HEVC Main */
+			if (rscreen->family >= CHIP_STONEY)
+				return (profile == PIPE_VIDEO_PROFILE_HEVC_MAIN ||
+					profile == PIPE_VIDEO_PROFILE_HEVC_MAIN_10);
+			else if (rscreen->family >= CHIP_CARRIZO)
+				return profile == PIPE_VIDEO_PROFILE_HEVC_MAIN;
+			return false;
+		case PIPE_VIDEO_FORMAT_JPEG:
+			if (rscreen->family < CHIP_CARRIZO || rscreen->family >= CHIP_VEGA10)
+				return false;
+			if (!(rscreen->info.drm_major == 3 && rscreen->info.drm_minor >= 19)) {
+				RVID_ERR("No MJPEG support for the kernel version\n");
+				return false;
+			}
+			return true;
+		default:
+			return false;
+		}
+	case PIPE_VIDEO_CAP_NPOT_TEXTURES:
+		return 1;
+	case PIPE_VIDEO_CAP_MAX_WIDTH:
+		return (rscreen->family < CHIP_TONGA) ? 2048 : 4096;
+	case PIPE_VIDEO_CAP_MAX_HEIGHT:
+		return (rscreen->family < CHIP_TONGA) ? 1152 : 4096;
+	case PIPE_VIDEO_CAP_PREFERED_FORMAT:
+		if (profile == PIPE_VIDEO_PROFILE_HEVC_MAIN_10)
+			return PIPE_FORMAT_P016;
+		else
+			return PIPE_FORMAT_NV12;
+
+	case PIPE_VIDEO_CAP_PREFERS_INTERLACED:
+	case PIPE_VIDEO_CAP_SUPPORTS_INTERLACED:
+		if (rscreen->family < CHIP_PALM) {
+			/* MPEG2 only with shaders and no support for
+			   interlacing on R6xx style UVD */
+			return codec != PIPE_VIDEO_FORMAT_MPEG12 &&
+			       rscreen->family > CHIP_RV770;
+		} else {
+			enum pipe_video_format format = u_reduce_video_profile(profile);
+
+			if (format == PIPE_VIDEO_FORMAT_HEVC)
+				return false; //The firmware doesn't support interlaced HEVC.
+			else if (format == PIPE_VIDEO_FORMAT_JPEG)
+				return false;
+			return true;
+		}
+	case PIPE_VIDEO_CAP_SUPPORTS_PROGRESSIVE:
+		return true;
+	case PIPE_VIDEO_CAP_MAX_LEVEL:
+		switch (profile) {
+		case PIPE_VIDEO_PROFILE_MPEG1:
+			return 0;
+		case PIPE_VIDEO_PROFILE_MPEG2_SIMPLE:
+		case PIPE_VIDEO_PROFILE_MPEG2_MAIN:
+			return 3;
+		case PIPE_VIDEO_PROFILE_MPEG4_SIMPLE:
+			return 3;
+		case PIPE_VIDEO_PROFILE_MPEG4_ADVANCED_SIMPLE:
+			return 5;
+		case PIPE_VIDEO_PROFILE_VC1_SIMPLE:
+			return 1;
+		case PIPE_VIDEO_PROFILE_VC1_MAIN:
+			return 2;
+		case PIPE_VIDEO_PROFILE_VC1_ADVANCED:
+			return 4;
+		case PIPE_VIDEO_PROFILE_MPEG4_AVC_BASELINE:
+		case PIPE_VIDEO_PROFILE_MPEG4_AVC_MAIN:
+		case PIPE_VIDEO_PROFILE_MPEG4_AVC_HIGH:
+			return (rscreen->family < CHIP_TONGA) ? 41 : 52;
+		case PIPE_VIDEO_PROFILE_HEVC_MAIN:
+		case PIPE_VIDEO_PROFILE_HEVC_MAIN_10:
+			return 186;
+		default:
+			return 0;
+		}
+	default:
+		return 0;
+	}
+}
+
+boolean rvid_is_format_supported(struct pipe_screen *screen,
+				 enum pipe_format format,
+				 enum pipe_video_profile profile,
+				 enum pipe_video_entrypoint entrypoint)
+{
+	/* HEVC 10 bit decoding should use P016 instead of NV12 if possible */
+	if (profile == PIPE_VIDEO_PROFILE_HEVC_MAIN_10)
+		return (format == PIPE_FORMAT_NV12) ||
+			(format == PIPE_FORMAT_P016);
+
+	/* we can only handle this one with UVD */
+	if (profile != PIPE_VIDEO_PROFILE_UNKNOWN)
+		return format == PIPE_FORMAT_NV12;
+
+	return vl_video_buffer_is_format_supported(screen, format, profile, entrypoint);
+}
diff --git a/src/gallium/drivers/r600/radeon_video.h b/src/gallium/drivers/r600/radeon_video.h
new file mode 100644
index 00000000000..3347c4ebced
--- /dev/null
+++ b/src/gallium/drivers/r600/radeon_video.h
@@ -0,0 +1,85 @@
+/**************************************************************************
+ *
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+/*
+ * Authors:
+ *      Christian König <[email protected]>
+ *
+ */
+
+#ifndef RADEON_VIDEO_H
+#define RADEON_VIDEO_H
+
+#include "radeon/radeon_winsys.h"
+#include "vl/vl_video_buffer.h"
+
+#define RVID_ERR(fmt, args...) \
+	fprintf(stderr, "EE %s:%d %s UVD - "fmt, __FILE__, __LINE__, __func__, ##args)
+
+/* video buffer representation */
+struct rvid_buffer
+{
+	unsigned		usage;
+	struct r600_resource	*res;
+};
+
+/* generate an stream handle */
+unsigned rvid_alloc_stream_handle(void);
+
+/* create a buffer in the winsys */
+bool rvid_create_buffer(struct pipe_screen *screen, struct rvid_buffer *buffer,
+			unsigned size, unsigned usage);
+
+/* destroy a buffer */
+void rvid_destroy_buffer(struct rvid_buffer *buffer);
+
+/* reallocate a buffer, preserving its content */
+bool rvid_resize_buffer(struct pipe_screen *screen, struct radeon_winsys_cs *cs,
+			struct rvid_buffer *new_buf, unsigned new_size);
+
+/* clear the buffer with zeros */
+void rvid_clear_buffer(struct pipe_context *context, struct rvid_buffer* buffer);
+
+/* join surfaces into the same buffer with identical tiling params
+   sumup their sizes and replace the backend buffers with a single bo */
+void rvid_join_surfaces(struct r600_common_context *rctx,
+                        struct pb_buffer** buffers[VL_NUM_COMPONENTS],
+                        struct radeon_surf *surfaces[VL_NUM_COMPONENTS]);
+
+/* returns supported codecs and other parameters */
+int rvid_get_video_param(struct pipe_screen *screen,
+                         enum pipe_video_profile profile,
+                         enum pipe_video_entrypoint entrypoint,
+                         enum pipe_video_cap param);
+
+/* the hardware only supports NV12 */
+boolean rvid_is_format_supported(struct pipe_screen *screen,
+                                 enum pipe_format format,
+                                 enum pipe_video_profile profile,
+                                 enum pipe_video_entrypoint entrypoint);
+
+#endif // RADEON_VIDEO_H