r600g: implement compression for MSAA colorbuffers for evergreen

This adds the FMASK and CMASK buffers. They share the same resource
with color data.

COMPRESSION and FAST_CLEAR are always enabled if both FMASK and CMASK are
allocated. We initialize the CMASK to a "compressed" state (not "fast cleared"),
so that we can keep FAST_CLEAR enabled all the time.

Both FMASK and CMASK must be present at the moment. If either one is missing,
the other one is not used.

v2: add cayman regs in the list

Reviewed-by: Jerome Glisse <[email protected]>

12 files changed, 529 insertions, 19 deletions

diff --git a/src/gallium/auxiliary/util/u_blitter.c b/src/gallium/auxiliary/util/u_blitter.c
index 25c7119b9f2..ad4ccd9eb46 100644
--- a/src/gallium/auxiliary/util/u_blitter.c
+++ b/src/gallium/auxiliary/util/u_blitter.c
@@ -1553,3 +1553,48 @@ void util_blitter_custom_resolve_color(struct blitter_context *blitter,
    pipe_surface_reference(&srcsurf, NULL);
    pipe_surface_reference(&dstsurf, NULL);
 }
+
+void util_blitter_custom_color(struct blitter_context *blitter,
+                               struct pipe_surface *dstsurf,
+                               void *custom_blend)
+{
+   struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
+   struct pipe_context *pipe = ctx->base.pipe;
+   struct pipe_framebuffer_state fb_state;
+
+   assert(dstsurf->texture);
+   if (!dstsurf->texture)
+      return;
+
+   /* check the saved state */
+   blitter_set_running_flag(ctx);
+   blitter_check_saved_vertex_states(ctx);
+   blitter_check_saved_fragment_states(ctx);
+   blitter_check_saved_fb_state(ctx);
+
+   /* bind states */
+   pipe->bind_blend_state(pipe, custom_blend);
+   pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_stencil);
+   pipe->bind_fs_state(pipe, blitter_get_fs_col(ctx, 1, FALSE));
+   pipe->bind_vertex_elements_state(pipe, ctx->velem_state);
+   pipe->set_sample_mask(pipe, (1ull << MAX2(1, dstsurf->texture->nr_samples)) - 1);
+
+   /* set a framebuffer state */
+   fb_state.width = dstsurf->width;
+   fb_state.height = dstsurf->height;
+   fb_state.nr_cbufs = 1;
+   fb_state.cbufs[0] = dstsurf;
+   fb_state.zsbuf = 0;
+   pipe->set_framebuffer_state(pipe, &fb_state);
+   pipe->set_sample_mask(pipe, ~0);
+
+   blitter_set_common_draw_rect_state(ctx);
+   blitter_set_dst_dimensions(ctx, dstsurf->width, dstsurf->height);
+   blitter->draw_rectangle(blitter, 0, 0, dstsurf->width, dstsurf->height,
+                           0, 0, NULL);
+
+   blitter_restore_vertex_states(ctx);
+   blitter_restore_fragment_states(ctx);
+   blitter_restore_fb_state(ctx);
+   blitter_unset_running_flag(ctx);
+}
diff --git a/src/gallium/auxiliary/util/u_blitter.h b/src/gallium/auxiliary/util/u_blitter.h
index f227902c163..e06e8b12d53 100644
--- a/src/gallium/auxiliary/util/u_blitter.h
+++ b/src/gallium/auxiliary/util/u_blitter.h
@@ -308,6 +308,11 @@ void util_blitter_custom_depth_stencil(struct blitter_context *blitter,
 				       unsigned sample_mask,
 				       void *dsa_stage, float depth);
 
+/* Used by r600g for color decompression. */
+void util_blitter_custom_color(struct blitter_context *blitter,
+                               struct pipe_surface *dstsurf,
+                               void *custom_blend);
+
 /* Used by r600g for MSAA color resolve. */
 void util_blitter_custom_resolve_color(struct blitter_context *blitter,
                                        struct pipe_resource *dst,
diff --git a/src/gallium/drivers/r600/evergreen_hw_context.c b/src/gallium/drivers/r600/evergreen_hw_context.c
index f0c4ff7a482..d2f09498566 100644
--- a/src/gallium/drivers/r600/evergreen_hw_context.c
+++ b/src/gallium/drivers/r600/evergreen_hw_context.c
@@ -219,6 +219,14 @@ static const struct r600_reg evergreen_context_reg_list[] = {
 	{R_028C70_CB_COLOR0_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028C74_CB_COLOR0_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028C78_CB_COLOR0_DIM, 0, 0},
+	{R_028C7C_CB_COLOR0_CMASK, REG_FLAG_NEED_BO},
+	{R_028C80_CB_COLOR0_CMASK_SLICE},
+	{R_028C84_CB_COLOR0_FMASK, REG_FLAG_NEED_BO},
+	{R_028C88_CB_COLOR0_FMASK_SLICE},
+	{R_028C8C_CB_COLOR0_CLEAR_WORD0},
+	{R_028C90_CB_COLOR0_CLEAR_WORD1},
+	{R_028C94_CB_COLOR0_CLEAR_WORD2},
+	{R_028C98_CB_COLOR0_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028C9C_CB_COLOR1_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028CA0_CB_COLOR1_PITCH, 0, 0},
@@ -227,6 +235,14 @@ static const struct r600_reg evergreen_context_reg_list[] = {
 	{R_028CAC_CB_COLOR1_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028CB0_CB_COLOR1_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028CB4_CB_COLOR1_DIM, 0, 0},
+	{R_028CB8_CB_COLOR1_CMASK, REG_FLAG_NEED_BO, 0},
+	{R_028CBC_CB_COLOR1_CMASK_SLICE, 0, 0},
+	{R_028CC0_CB_COLOR1_FMASK, REG_FLAG_NEED_BO, 0},
+	{R_028CC4_CB_COLOR1_FMASK_SLICE, 0, 0},
+	{R_028CC8_CB_COLOR1_CLEAR_WORD0},
+	{R_028CCC_CB_COLOR1_CLEAR_WORD1},
+	{R_028CD0_CB_COLOR1_CLEAR_WORD2},
+	{R_028CD4_CB_COLOR1_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028CD8_CB_COLOR2_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028CDC_CB_COLOR2_PITCH, 0, 0},
@@ -235,6 +251,14 @@ static const struct r600_reg evergreen_context_reg_list[] = {
 	{R_028CE8_CB_COLOR2_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028CEC_CB_COLOR2_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028CF0_CB_COLOR2_DIM, 0, 0},
+	{R_028CF4_CB_COLOR2_CMASK, REG_FLAG_NEED_BO, 0},
+	{R_028CF8_CB_COLOR2_CMASK_SLICE, 0, 0},
+	{R_028CFC_CB_COLOR2_FMASK, REG_FLAG_NEED_BO, 0},
+	{R_028D00_CB_COLOR2_FMASK_SLICE, 0, 0},
+	{R_028D04_CB_COLOR2_CLEAR_WORD0},
+	{R_028D08_CB_COLOR2_CLEAR_WORD1},
+	{R_028D0C_CB_COLOR2_CLEAR_WORD2},
+	{R_028D10_CB_COLOR2_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028D14_CB_COLOR3_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028D18_CB_COLOR3_PITCH, 0, 0},
@@ -243,6 +267,14 @@ static const struct r600_reg evergreen_context_reg_list[] = {
 	{R_028D24_CB_COLOR3_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028D28_CB_COLOR3_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028D2C_CB_COLOR3_DIM, 0, 0},
+	{R_028D30_CB_COLOR3_CMASK, REG_FLAG_NEED_BO},
+	{R_028D34_CB_COLOR3_CMASK_SLICE},
+	{R_028D38_CB_COLOR3_FMASK, REG_FLAG_NEED_BO},
+	{R_028D3C_CB_COLOR3_FMASK_SLICE},
+	{R_028D40_CB_COLOR3_CLEAR_WORD0},
+	{R_028D44_CB_COLOR3_CLEAR_WORD1},
+	{R_028D48_CB_COLOR3_CLEAR_WORD2},
+	{R_028D4C_CB_COLOR3_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028D50_CB_COLOR4_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028D54_CB_COLOR4_PITCH, 0, 0},
@@ -251,6 +283,14 @@ static const struct r600_reg evergreen_context_reg_list[] = {
 	{R_028D60_CB_COLOR4_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028D64_CB_COLOR4_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028D68_CB_COLOR4_DIM, 0, 0},
+	{R_028D6C_CB_COLOR4_CMASK, REG_FLAG_NEED_BO},
+	{R_028D70_CB_COLOR4_CMASK_SLICE},
+	{R_028D74_CB_COLOR4_FMASK, REG_FLAG_NEED_BO},
+	{R_028D78_CB_COLOR4_FMASK_SLICE},
+	{R_028D7C_CB_COLOR4_CLEAR_WORD0},
+	{R_028D80_CB_COLOR4_CLEAR_WORD1},
+	{R_028D84_CB_COLOR4_CLEAR_WORD2},
+	{R_028D88_CB_COLOR4_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028D8C_CB_COLOR5_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028D90_CB_COLOR5_PITCH, 0, 0},
@@ -259,6 +299,14 @@ static const struct r600_reg evergreen_context_reg_list[] = {
 	{R_028D9C_CB_COLOR5_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028DA0_CB_COLOR5_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028DA4_CB_COLOR5_DIM, 0, 0},
+	{R_028DA8_CB_COLOR5_CMASK, REG_FLAG_NEED_BO},
+	{R_028DAC_CB_COLOR5_CMASK_SLICE},
+	{R_028DB0_CB_COLOR5_FMASK, REG_FLAG_NEED_BO},
+	{R_028DB4_CB_COLOR5_FMASK_SLICE},
+	{R_028DB8_CB_COLOR5_CLEAR_WORD0},
+	{R_028DBC_CB_COLOR5_CLEAR_WORD1},
+	{R_028DC0_CB_COLOR5_CLEAR_WORD2},
+	{R_028DC4_CB_COLOR5_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028DC8_CB_COLOR6_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028DCC_CB_COLOR6_PITCH, 0, 0},
@@ -267,6 +315,14 @@ static const struct r600_reg evergreen_context_reg_list[] = {
 	{R_028DD8_CB_COLOR6_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028DDC_CB_COLOR6_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028DE0_CB_COLOR6_DIM, 0, 0},
+	{R_028DE4_CB_COLOR6_CMASK, REG_FLAG_NEED_BO},
+	{R_028DE8_CB_COLOR6_CMASK_SLICE},
+	{R_028DEC_CB_COLOR6_FMASK, REG_FLAG_NEED_BO},
+	{R_028DF0_CB_COLOR6_FMASK_SLICE},
+	{R_028DF4_CB_COLOR6_CLEAR_WORD0},
+	{R_028DF8_CB_COLOR6_CLEAR_WORD1},
+	{R_028DFC_CB_COLOR6_CLEAR_WORD2},
+	{R_028E00_CB_COLOR6_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028E04_CB_COLOR7_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028E08_CB_COLOR7_PITCH, 0, 0},
@@ -275,6 +331,14 @@ static const struct r600_reg evergreen_context_reg_list[] = {
 	{R_028E14_CB_COLOR7_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028E18_CB_COLOR7_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028E1C_CB_COLOR7_DIM, 0, 0},
+	{R_028E20_CB_COLOR7_CMASK, REG_FLAG_NEED_BO},
+	{R_028E24_CB_COLOR7_CMASK_SLICE},
+	{R_028E28_CB_COLOR7_FMASK, REG_FLAG_NEED_BO},
+	{R_028E2C_CB_COLOR7_FMASK_SLICE},
+	{R_028E30_CB_COLOR7_CLEAR_WORD0},
+	{R_028E34_CB_COLOR7_CLEAR_WORD1},
+	{R_028E38_CB_COLOR7_CLEAR_WORD2},
+	{R_028E3C_CB_COLOR7_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028E40_CB_COLOR8_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028E44_CB_COLOR8_PITCH, 0, 0},
@@ -500,6 +564,14 @@ static const struct r600_reg cayman_context_reg_list[] = {
 	{R_028C70_CB_COLOR0_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028C74_CB_COLOR0_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028C78_CB_COLOR0_DIM, 0, 0},
+	{R_028C7C_CB_COLOR0_CMASK, REG_FLAG_NEED_BO},
+	{R_028C80_CB_COLOR0_CMASK_SLICE},
+	{R_028C84_CB_COLOR0_FMASK, REG_FLAG_NEED_BO},
+	{R_028C88_CB_COLOR0_FMASK_SLICE},
+	{R_028C8C_CB_COLOR0_CLEAR_WORD0},
+	{R_028C90_CB_COLOR0_CLEAR_WORD1},
+	{R_028C94_CB_COLOR0_CLEAR_WORD2},
+	{R_028C98_CB_COLOR0_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028C9C_CB_COLOR1_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028CA0_CB_COLOR1_PITCH, 0, 0},
@@ -508,6 +580,14 @@ static const struct r600_reg cayman_context_reg_list[] = {
 	{R_028CAC_CB_COLOR1_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028CB0_CB_COLOR1_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028CB4_CB_COLOR1_DIM, 0, 0},
+	{R_028CB8_CB_COLOR1_CMASK, REG_FLAG_NEED_BO, 0},
+	{R_028CBC_CB_COLOR1_CMASK_SLICE, 0, 0},
+	{R_028CC0_CB_COLOR1_FMASK, REG_FLAG_NEED_BO, 0},
+	{R_028CC4_CB_COLOR1_FMASK_SLICE, 0, 0},
+	{R_028CC8_CB_COLOR1_CLEAR_WORD0},
+	{R_028CCC_CB_COLOR1_CLEAR_WORD1},
+	{R_028CD0_CB_COLOR1_CLEAR_WORD2},
+	{R_028CD4_CB_COLOR1_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028CD8_CB_COLOR2_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028CDC_CB_COLOR2_PITCH, 0, 0},
@@ -516,6 +596,14 @@ static const struct r600_reg cayman_context_reg_list[] = {
 	{R_028CE8_CB_COLOR2_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028CEC_CB_COLOR2_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028CF0_CB_COLOR2_DIM, 0, 0},
+	{R_028CF4_CB_COLOR2_CMASK, REG_FLAG_NEED_BO, 0},
+	{R_028CF8_CB_COLOR2_CMASK_SLICE, 0, 0},
+	{R_028CFC_CB_COLOR2_FMASK, REG_FLAG_NEED_BO, 0},
+	{R_028D00_CB_COLOR2_FMASK_SLICE, 0, 0},
+	{R_028D04_CB_COLOR2_CLEAR_WORD0},
+	{R_028D08_CB_COLOR2_CLEAR_WORD1},
+	{R_028D0C_CB_COLOR2_CLEAR_WORD2},
+	{R_028D10_CB_COLOR2_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028D14_CB_COLOR3_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028D18_CB_COLOR3_PITCH, 0, 0},
@@ -524,6 +612,14 @@ static const struct r600_reg cayman_context_reg_list[] = {
 	{R_028D24_CB_COLOR3_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028D28_CB_COLOR3_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028D2C_CB_COLOR3_DIM, 0, 0},
+	{R_028D30_CB_COLOR3_CMASK, REG_FLAG_NEED_BO},
+	{R_028D34_CB_COLOR3_CMASK_SLICE},
+	{R_028D38_CB_COLOR3_FMASK, REG_FLAG_NEED_BO},
+	{R_028D3C_CB_COLOR3_FMASK_SLICE},
+	{R_028D40_CB_COLOR3_CLEAR_WORD0},
+	{R_028D44_CB_COLOR3_CLEAR_WORD1},
+	{R_028D48_CB_COLOR3_CLEAR_WORD2},
+	{R_028D4C_CB_COLOR3_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028D50_CB_COLOR4_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028D54_CB_COLOR4_PITCH, 0, 0},
@@ -532,6 +628,14 @@ static const struct r600_reg cayman_context_reg_list[] = {
 	{R_028D60_CB_COLOR4_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028D64_CB_COLOR4_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028D68_CB_COLOR4_DIM, 0, 0},
+	{R_028D6C_CB_COLOR4_CMASK, REG_FLAG_NEED_BO},
+	{R_028D70_CB_COLOR4_CMASK_SLICE},
+	{R_028D74_CB_COLOR4_FMASK, REG_FLAG_NEED_BO},
+	{R_028D78_CB_COLOR4_FMASK_SLICE},
+	{R_028D7C_CB_COLOR4_CLEAR_WORD0},
+	{R_028D80_CB_COLOR4_CLEAR_WORD1},
+	{R_028D84_CB_COLOR4_CLEAR_WORD2},
+	{R_028D88_CB_COLOR4_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028D8C_CB_COLOR5_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028D90_CB_COLOR5_PITCH, 0, 0},
@@ -540,6 +644,14 @@ static const struct r600_reg cayman_context_reg_list[] = {
 	{R_028D9C_CB_COLOR5_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028DA0_CB_COLOR5_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028DA4_CB_COLOR5_DIM, 0, 0},
+	{R_028DA8_CB_COLOR5_CMASK, REG_FLAG_NEED_BO},
+	{R_028DAC_CB_COLOR5_CMASK_SLICE},
+	{R_028DB0_CB_COLOR5_FMASK, REG_FLAG_NEED_BO},
+	{R_028DB4_CB_COLOR5_FMASK_SLICE},
+	{R_028DB8_CB_COLOR5_CLEAR_WORD0},
+	{R_028DBC_CB_COLOR5_CLEAR_WORD1},
+	{R_028DC0_CB_COLOR5_CLEAR_WORD2},
+	{R_028DC4_CB_COLOR5_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028DC8_CB_COLOR6_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028DCC_CB_COLOR6_PITCH, 0, 0},
@@ -548,6 +660,14 @@ static const struct r600_reg cayman_context_reg_list[] = {
 	{R_028DD8_CB_COLOR6_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028DDC_CB_COLOR6_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028DE0_CB_COLOR6_DIM, 0, 0},
+	{R_028DE4_CB_COLOR6_CMASK, REG_FLAG_NEED_BO},
+	{R_028DE8_CB_COLOR6_CMASK_SLICE},
+	{R_028DEC_CB_COLOR6_FMASK, REG_FLAG_NEED_BO},
+	{R_028DF0_CB_COLOR6_FMASK_SLICE},
+	{R_028DF4_CB_COLOR6_CLEAR_WORD0},
+	{R_028DF8_CB_COLOR6_CLEAR_WORD1},
+	{R_028DFC_CB_COLOR6_CLEAR_WORD2},
+	{R_028E00_CB_COLOR6_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028E04_CB_COLOR7_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028E08_CB_COLOR7_PITCH, 0, 0},
@@ -556,6 +676,14 @@ static const struct r600_reg cayman_context_reg_list[] = {
 	{R_028E14_CB_COLOR7_INFO, REG_FLAG_NEED_BO, 0},
 	{R_028E18_CB_COLOR7_ATTRIB, REG_FLAG_NEED_BO, 0},
 	{R_028E1C_CB_COLOR7_DIM, 0, 0},
+	{R_028E20_CB_COLOR7_CMASK, REG_FLAG_NEED_BO},
+	{R_028E24_CB_COLOR7_CMASK_SLICE},
+	{R_028E28_CB_COLOR7_FMASK, REG_FLAG_NEED_BO},
+	{R_028E2C_CB_COLOR7_FMASK_SLICE},
+	{R_028E30_CB_COLOR7_CLEAR_WORD0},
+	{R_028E34_CB_COLOR7_CLEAR_WORD1},
+	{R_028E38_CB_COLOR7_CLEAR_WORD2},
+	{R_028E3C_CB_COLOR7_CLEAR_WORD3},
 	{GROUP_FORCE_NEW_BLOCK, 0, 0},
 	{R_028E40_CB_COLOR8_BASE, REG_FLAG_NEED_BO, 0},
 	{R_028E44_CB_COLOR8_PITCH, 0, 0},
diff --git a/src/gallium/drivers/r600/evergreen_state.c b/src/gallium/drivers/r600/evergreen_state.c
index e12706e856b..35bc391fe80 100644
--- a/src/gallium/drivers/r600/evergreen_state.c
+++ b/src/gallium/drivers/r600/evergreen_state.c
@@ -1250,8 +1250,8 @@ void evergreen_init_color_surface(struct r600_context *rctx,
 	unsigned pitch, slice;
 	unsigned color_info, color_attrib, color_dim = 0;
 	unsigned format, swap, ntype, endian;
-	uint64_t offset;
-	unsigned tile_type, macro_aspect, tile_split, bankh, bankw, nbanks;
+	uint64_t offset, base_offset;
+	unsigned tile_type, macro_aspect, tile_split, bankh, bankw, fmask_bankh, nbanks;
 	const struct util_format_description *desc;
 	int i;
 	bool blend_clamp = 0, blend_bypass = 0;
@@ -1296,10 +1296,12 @@ void evergreen_init_color_surface(struct r600_context *rctx,
 	macro_aspect = rtex->surface.mtilea;
 	bankw = rtex->surface.bankw;
 	bankh = rtex->surface.bankh;
+	fmask_bankh = rtex->fmask_bank_height;
 	tile_split = eg_tile_split(tile_split);
 	macro_aspect = eg_macro_tile_aspect(macro_aspect);
 	bankw = eg_bank_wh(bankw);
 	bankh = eg_bank_wh(bankh);
+	fmask_bankh = eg_bank_wh(fmask_bankh);
 
 	/* 128 bit formats require tile type = 1 */
 	if (rscreen->chip_class == CAYMAN) {
@@ -1319,7 +1321,8 @@ void evergreen_init_color_surface(struct r600_context *rctx,
 			S_028C74_BANK_WIDTH(bankw) |
 			S_028C74_BANK_HEIGHT(bankh) |
 			S_028C74_MACRO_TILE_ASPECT(macro_aspect) |
-			S_028C74_NON_DISP_TILING_ORDER(tile_type);
+			S_028C74_NON_DISP_TILING_ORDER(tile_type) |
+		        S_028C74_FMASK_BANK_HEIGHT(fmask_bankh);
 
 	ntype = V_028C70_NUMBER_UNORM;
 	if (desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB)
@@ -1393,11 +1396,14 @@ void evergreen_init_color_surface(struct r600_context *rctx,
 		surf->export_16bpc = true;
 	}
 
-	offset += r600_resource_va(rctx->context.screen, pipe_tex);
-	offset >>= 8;
+	if (rtex->fmask_size && rtex->cmask_size) {
+		color_info |= S_028C70_COMPRESSION(1) | S_028C70_FAST_CLEAR(1);
+	}
+
+	base_offset = r600_resource_va(rctx->context.screen, pipe_tex);
 
 	/* XXX handle enabling of CB beyond BASE8 which has different offset */
-	surf->cb_color_base = offset;
+	surf->cb_color_base = (base_offset + offset) >> 8;
 	surf->cb_color_dim = color_dim;
 	surf->cb_color_info = color_info;
 	surf->cb_color_pitch = S_028C64_PITCH_TILE_MAX(pitch);
@@ -1409,6 +1415,15 @@ void evergreen_init_color_surface(struct r600_context *rctx,
 				      S_028C6C_SLICE_MAX(surf->base.u.tex.last_layer);
 	}
 	surf->cb_color_attrib = color_attrib;
+	if (rtex->fmask_size && rtex->cmask_size) {
+		surf->cb_color_fmask = (base_offset + rtex->fmask_offset) >> 8;
+		surf->cb_color_cmask = (base_offset + rtex->cmask_offset) >> 8;
+	} else {
+		surf->cb_color_fmask = surf->cb_color_base;
+		surf->cb_color_cmask = surf->cb_color_base;
+	}
+	surf->cb_color_fmask_slice = S_028C88_TILE_MAX(slice);
+	surf->cb_color_cmask_slice = S_028C80_TILE_MAX(rtex->cmask_slice_tile_max);
 
 	surf->color_initialized = true;
 }
@@ -1631,6 +1646,7 @@ static void evergreen_set_framebuffer_state(struct pipe_context *ctx,
 	struct r600_pipe_state *rstate = CALLOC_STRUCT(r600_pipe_state);
 	struct r600_surface *surf;
 	struct r600_resource *res;
+	struct r600_texture *rtex;
 	uint32_t tl, br, i, nr_samples;
 
 	if (rstate == NULL)
@@ -1648,10 +1664,12 @@ static void evergreen_set_framebuffer_state(struct pipe_context *ctx,
 	rctx->nr_cbufs = state->nr_cbufs;
 	rctx->cb0_is_integer = state->nr_cbufs &&
 			       util_format_is_pure_integer(state->cbufs[0]->format);
+	rctx->compressed_cb_mask = 0;
 
 	for (i = 0; i < state->nr_cbufs; i++) {
 		surf = (struct r600_surface*)state->cbufs[i];
 		res = (struct r600_resource*)surf->base.texture;
+		rtex = (struct r600_texture*)res;
 
 		if (!surf->color_initialized) {
 			evergreen_init_color_surface(rctx, surf);
@@ -1675,6 +1693,18 @@ static void evergreen_set_framebuffer_state(struct pipe_context *ctx,
 					surf->cb_color_view);
 		r600_pipe_state_add_reg_bo(rstate, R_028C74_CB_COLOR0_ATTRIB + i * 0x3C,
 					   surf->cb_color_attrib, res, RADEON_USAGE_READWRITE);
+		r600_pipe_state_add_reg_bo(rstate, R_028C7C_CB_COLOR0_CMASK + i * 0x3c,
+					   surf->cb_color_cmask, res, RADEON_USAGE_READWRITE);
+		r600_pipe_state_add_reg(rstate, R_028C80_CB_COLOR0_CMASK_SLICE + i * 0x3c,
+					surf->cb_color_cmask_slice);
+		r600_pipe_state_add_reg_bo(rstate,  R_028C84_CB_COLOR0_FMASK + i * 0x3c,
+					   surf->cb_color_fmask, res, RADEON_USAGE_READWRITE);
+		r600_pipe_state_add_reg(rstate, R_028C88_CB_COLOR0_FMASK_SLICE + i * 0x3c,
+					surf->cb_color_fmask_slice);
+
+		if (rtex->fmask_size && rtex->cmask_size) {
+			rctx->compressed_cb_mask |= 1 << i;
+		}
 	}
 	/* set CB_COLOR1_INFO for possible dual-src blending */
 	if (i == 1 && !((struct r600_texture*)res)->is_rat) {
@@ -3080,6 +3110,18 @@ void *evergreen_create_resolve_blend(struct r600_context *rctx)
 	return rstate;
 }
 
+void *evergreen_create_decompress_blend(struct r600_context *rctx)
+{
+	struct pipe_blend_state blend;
+	struct r600_pipe_state *rstate;
+
+	memset(&blend, 0, sizeof(blend));
+	blend.independent_blend_enable = true;
+	blend.rt[0].colormask = 0xf;
+	rstate = evergreen_create_blend_state_mode(&rctx->context, &blend, V_028808_CB_DECOMPRESS);
+	return rstate;
+}
+
 void *evergreen_create_db_flush_dsa(struct r600_context *rctx)
 {
 	struct pipe_depth_stencil_alpha_state dsa = {{0}};
diff --git a/src/gallium/drivers/r600/evergreend.h b/src/gallium/drivers/r600/evergreend.h
index d1b3cd33bb7..0c56aa6f693 100644
--- a/src/gallium/drivers/r600/evergreend.h
+++ b/src/gallium/drivers/r600/evergreend.h
@@ -377,6 +377,7 @@
 #define   S_028C74_BANK_WIDTH(x)                       (((x) & 0x3) << 13)
 #define   S_028C74_BANK_HEIGHT(x)                      (((x) & 0x3) << 16)
 #define   S_028C74_MACRO_TILE_ASPECT(x)                (((x) & 0x3) << 19)
+#define   S_028C74_FMASK_BANK_HEIGHT(x)                (((x) & 0x3) << 22)
 
 #define R_028C78_CB_COLOR0_DIM                         0x028C78
 #define   S_028C78_WIDTH_MAX(x)                        (((x) & 0xFFFF) << 0)
@@ -386,15 +387,6 @@
 #define   G_028C78_HEIGHT_MAX(x)                       (((x) >> 16) & 0xFFFF)
 #define   C_028C78_HEIGHT_MAX                          0x0000FFFF
 
-#define R_028C7C_CB_COLOR0_CMASK                         0x028C7C
-#define R_028C80_CB_COLOR0_CMASK_SLICE                   0x028C80
-#define R_028C84_CB_COLOR0_FMASK                         0x028C84
-#define R_028C88_CB_COLOR0_FMASK_SLICE                   0x028C88
-
-#define R_028C8C_CB_COLOR0_CLEAR_WORD0                   0x028C8C
-#define R_028C90_CB_COLOR0_CLEAR_WORD1                   0x028C90
-#define R_028C94_CB_COLOR0_CLEAR_WORD2                   0x028C94
-#define R_028C98_CB_COLOR0_CLEAR_WORD3                   0x028C98
 
 /* alpha same */
 #define R_028410_SX_ALPHA_TEST_CONTROL               0x028410
@@ -1978,6 +1970,16 @@
 #define   S_028C6C_SLICE_MAX(x)                        (((x) & 0x7FF) << 13)
 #define   G_028C6C_SLICE_MAX(x)                        (((x) >> 13) & 0x7FF)
 #define   C_028C6C_SLICE_MAX                           0xFF001FFF
+#define R_028C7C_CB_COLOR0_CMASK                         0x028C7C
+#define R_028C80_CB_COLOR0_CMASK_SLICE                   0x028C80
+#define   S_028C80_TILE_MAX(x)				(((x) & 0x3FFF) << 0)
+#define R_028C84_CB_COLOR0_FMASK                         0x028C84
+#define R_028C88_CB_COLOR0_FMASK_SLICE                   0x028C88
+#define   S_028C88_TILE_MAX(x)				(((x) & 0x3FFFFF) << 0)
+#define R_028C8C_CB_COLOR0_CLEAR_WORD0                   0x028C8C
+#define R_028C90_CB_COLOR0_CLEAR_WORD1                   0x028C90
+#define R_028C94_CB_COLOR0_CLEAR_WORD2                   0x028C94
+#define R_028C98_CB_COLOR0_CLEAR_WORD3                   0x028C98
 #define R_028C9C_CB_COLOR1_BASE                      0x00028C9C
 #define R_028CA0_CB_COLOR1_PITCH                     0x00028CA0
 #define R_028CA4_CB_COLOR1_SLICE                     0x00028CA4
@@ -1985,6 +1987,14 @@
 #define R_028CAC_CB_COLOR1_INFO                      0x00028CAC
 #define R_028CB0_CB_COLOR1_ATTRIB                    0x00028CB0
 #define R_028CB4_CB_COLOR1_DIM                       0x00028CB4
+#define R_028CB8_CB_COLOR1_CMASK                         0x028CB8
+#define R_028CBC_CB_COLOR1_CMASK_SLICE                   0x028CBC
+#define R_028CC0_CB_COLOR1_FMASK                         0x028CC0
+#define R_028CC4_CB_COLOR1_FMASK_SLICE                   0x028CC4
+#define R_028CC8_CB_COLOR1_CLEAR_WORD0                   0x028CC8
+#define R_028CCC_CB_COLOR1_CLEAR_WORD1                   0x028CCC
+#define R_028CD0_CB_COLOR1_CLEAR_WORD2                   0x028CD0
+#define R_028CD4_CB_COLOR1_CLEAR_WORD3                   0x028CD4
 #define R_028CD8_CB_COLOR2_BASE                      0x00028CD8
 #define R_028CDC_CB_COLOR2_PITCH                     0x00028CDC
 #define R_028CE0_CB_COLOR2_SLICE                     0x00028CE0
@@ -1992,6 +2002,14 @@
 #define R_028CE8_CB_COLOR2_INFO                      0x00028CE8
 #define R_028CEC_CB_COLOR2_ATTRIB                    0x00028CEC
 #define R_028CF0_CB_COLOR2_DIM                       0x00028CF0
+#define R_028CF4_CB_COLOR2_CMASK                         0x028CF4
+#define R_028CF8_CB_COLOR2_CMASK_SLICE                   0x028CF8
+#define R_028CFC_CB_COLOR2_FMASK                         0x028CFC
+#define R_028D00_CB_COLOR2_FMASK_SLICE                   0x028D00
+#define R_028D04_CB_COLOR2_CLEAR_WORD0                   0x028D04
+#define R_028D08_CB_COLOR2_CLEAR_WORD1                   0x028D08
+#define R_028D0C_CB_COLOR2_CLEAR_WORD2                   0x028D0C
+#define R_028D10_CB_COLOR2_CLEAR_WORD3                   0x028D10
 #define R_028D14_CB_COLOR3_BASE                      0x00028D14
 #define R_028D18_CB_COLOR3_PITCH                     0x00028D18
 #define R_028D1C_CB_COLOR3_SLICE                     0x00028D1C
@@ -1999,6 +2017,14 @@
 #define R_028D24_CB_COLOR3_INFO                      0x00028D24
 #define R_028D28_CB_COLOR3_ATTRIB                    0x00028D28
 #define R_028D2C_CB_COLOR3_DIM                       0x00028D2C
+#define R_028D30_CB_COLOR3_CMASK                         0x028D30
+#define R_028D34_CB_COLOR3_CMASK_SLICE                   0x028D34
+#define R_028D38_CB_COLOR3_FMASK                         0x028D38
+#define R_028D3C_CB_COLOR3_FMASK_SLICE                   0x028D3C
+#define R_028D40_CB_COLOR3_CLEAR_WORD0                   0x028D40
+#define R_028D44_CB_COLOR3_CLEAR_WORD1                   0x028D44
+#define R_028D48_CB_COLOR3_CLEAR_WORD2                   0x028D48
+#define R_028D4C_CB_COLOR3_CLEAR_WORD3                   0x028D4C
 #define R_028D50_CB_COLOR4_BASE                      0x00028D50
 #define R_028D54_CB_COLOR4_PITCH                     0x00028D54
 #define R_028D58_CB_COLOR4_SLICE                     0x00028D58
@@ -2006,6 +2032,14 @@
 #define R_028D60_CB_COLOR4_INFO                      0x00028D60
 #define R_028D64_CB_COLOR4_ATTRIB                    0x00028D64
 #define R_028D68_CB_COLOR4_DIM                       0x00028D68
+#define R_028D6C_CB_COLOR4_CMASK                         0x028D6C
+#define R_028D70_CB_COLOR4_CMASK_SLICE                   0x028D70
+#define R_028D74_CB_COLOR4_FMASK                         0x028D74
+#define R_028D78_CB_COLOR4_FMASK_SLICE                   0x028D78
+#define R_028D7C_CB_COLOR4_CLEAR_WORD0                   0x028D7C
+#define R_028D80_CB_COLOR4_CLEAR_WORD1                   0x028D80
+#define R_028D84_CB_COLOR4_CLEAR_WORD2                   0x028D84
+#define R_028D88_CB_COLOR4_CLEAR_WORD3                   0x028D88
 #define R_028D8C_CB_COLOR5_BASE                      0x00028D8C
 #define R_028D90_CB_COLOR5_PITCH                     0x00028D90
 #define R_028D94_CB_COLOR5_SLICE                     0x00028D94
@@ -2013,6 +2047,14 @@
 #define R_028D9C_CB_COLOR5_INFO                      0x00028D9C
 #define R_028DA0_CB_COLOR5_ATTRIB                    0x00028DA0
 #define R_028DA4_CB_COLOR5_DIM                       0x00028DA4
+#define R_028DA8_CB_COLOR5_CMASK                         0x028DA8
+#define R_028DAC_CB_COLOR5_CMASK_SLICE                   0x028DAC
+#define R_028DB0_CB_COLOR5_FMASK                         0x028DB0
+#define R_028DB4_CB_COLOR5_FMASK_SLICE                   0x028DB4
+#define R_028DB8_CB_COLOR5_CLEAR_WORD0                   0x028DB8
+#define R_028DBC_CB_COLOR5_CLEAR_WORD1                   0x028DBC
+#define R_028DC0_CB_COLOR5_CLEAR_WORD2                   0x028DC0
+#define R_028DC4_CB_COLOR5_CLEAR_WORD3                   0x028DC4
 #define R_028DC8_CB_COLOR6_BASE                      0x00028DC8
 #define R_028DCC_CB_COLOR6_PITCH                     0x00028DCC
 #define R_028DD0_CB_COLOR6_SLICE                     0x00028DD0
@@ -2020,6 +2062,14 @@
 #define R_028DD8_CB_COLOR6_INFO                      0x00028DD8
 #define R_028DDC_CB_COLOR6_ATTRIB                    0x00028DDC
 #define R_028DE0_CB_COLOR6_DIM                       0x00028DE0
+#define R_028DE4_CB_COLOR6_CMASK                         0x028DE4
+#define R_028DE8_CB_COLOR6_CMASK_SLICE                   0x028DE8
+#define R_028DEC_CB_COLOR6_FMASK                         0x028DEC
+#define R_028DF0_CB_COLOR6_FMASK_SLICE                   0x028DF0
+#define R_028DF4_CB_COLOR6_CLEAR_WORD0                   0x028DF4
+#define R_028DF8_CB_COLOR6_CLEAR_WORD1                   0x028DF8
+#define R_028DFC_CB_COLOR6_CLEAR_WORD2                   0x028DFC
+#define R_028E00_CB_COLOR6_CLEAR_WORD3                   0x028E00
 #define R_028E04_CB_COLOR7_BASE                      0x00028E04
 #define R_028E08_CB_COLOR7_PITCH                     0x00028E08
 #define R_028E0C_CB_COLOR7_SLICE                     0x00028E0C
@@ -2027,6 +2077,14 @@
 #define R_028E14_CB_COLOR7_INFO                      0x00028E14
 #define R_028E18_CB_COLOR7_ATTRIB                    0x00028E18
 #define R_028E1C_CB_COLOR7_DIM                       0x00028E1C
+#define R_028E20_CB_COLOR7_CMASK                         0x028E20
+#define R_028E24_CB_COLOR7_CMASK_SLICE                   0x028E24
+#define R_028E28_CB_COLOR7_FMASK                         0x028E28
+#define R_028E2C_CB_COLOR7_FMASK_SLICE                   0x028E2C
+#define R_028E30_CB_COLOR7_CLEAR_WORD0                   0x028E30
+#define R_028E34_CB_COLOR7_CLEAR_WORD1                   0x028E34
+#define R_028E38_CB_COLOR7_CLEAR_WORD2                   0x028E38
+#define R_028E3C_CB_COLOR7_CLEAR_WORD3                   0x028E3C
 #define R_028E40_CB_COLOR8_BASE                      0x00028E40
 #define R_028E44_CB_COLOR8_PITCH                     0x00028E44
 #define R_028E48_CB_COLOR8_SLICE                     0x00028E48
diff --git a/src/gallium/drivers/r600/r600_blit.c b/src/gallium/drivers/r600/r600_blit.c
index c4ad0f20ed4..e9ed0740e65 100644
--- a/src/gallium/drivers/r600/r600_blit.c
+++ b/src/gallium/drivers/r600/r600_blit.c
@@ -243,6 +243,76 @@ void r600_decompress_depth_textures(struct r600_context *rctx,
 	}
 }
 
+static void r600_blit_decompress_color(struct pipe_context *ctx,
+		struct r600_texture *rtex,
+		unsigned first_level, unsigned last_level,
+		unsigned first_layer, unsigned last_layer)
+{
+	struct r600_context *rctx = (struct r600_context *)ctx;
+	unsigned layer, level, checked_last_layer, max_layer;
+
+	if (!rtex->dirty_level_mask)
+		return;
+
+	for (level = first_level; level <= last_level; level++) {
+		if (!(rtex->dirty_level_mask & (1 << level)))
+			continue;
+
+		/* The smaller the mipmap level, the less layers there are
+		 * as far as 3D textures are concerned. */
+		max_layer = u_max_layer(&rtex->resource.b.b, level);
+		checked_last_layer = last_layer < max_layer ? last_layer : max_layer;
+
+		for (layer = first_layer; layer <= checked_last_layer; layer++) {
+			struct pipe_surface *cbsurf, surf_tmpl;
+
+			surf_tmpl.format = rtex->resource.b.b.format;
+			surf_tmpl.u.tex.level = level;
+			surf_tmpl.u.tex.first_layer = layer;
+			surf_tmpl.u.tex.last_layer = layer;
+			surf_tmpl.usage = PIPE_BIND_RENDER_TARGET;
+			cbsurf = ctx->create_surface(ctx, &rtex->resource.b.b, &surf_tmpl);
+
+			r600_blitter_begin(ctx, R600_DECOMPRESS);
+			util_blitter_custom_color(rctx->blitter, cbsurf,
+						  rctx->custom_blend_decompress);
+			r600_blitter_end(ctx);
+
+			pipe_surface_reference(&cbsurf, NULL);
+		}
+
+		/* The texture will always be dirty if some layers or samples aren't flushed.
+		 * I don't think this case occurs often though. */
+		if (first_layer == 0 && last_layer == max_layer) {
+			rtex->dirty_level_mask &= ~(1 << level);
+		}
+	}
+}
+
+void r600_decompress_color_textures(struct r600_context *rctx,
+				    struct r600_samplerview_state *textures)
+{
+	unsigned i;
+	unsigned mask = textures->compressed_colortex_mask;
+
+	while (mask) {
+		struct pipe_sampler_view *view;
+		struct r600_texture *tex;
+
+		i = u_bit_scan(&mask);
+
+		view = &textures->views[i]->base;
+		assert(view);
+
+		tex = (struct r600_texture *)view->texture;
+		assert(tex->cmask_size && tex->fmask_size);
+
+		r600_blit_decompress_color(&rctx->context, tex,
+					   view->u.tex.first_level, view->u.tex.last_level,
+					   0, u_max_layer(&tex->resource.b.b, view->u.tex.first_level));
+	}
+}
+
 static void r600_copy_first_sample(struct pipe_context *ctx,
 				   const struct pipe_resolve_info *info)
 {
@@ -262,6 +332,11 @@ static void r600_copy_first_sample(struct pipe_context *ctx,
 					   info->src.layer, info->src.layer,
 					   0, 0);
 	}
+	if (rsrc->fmask_size && rsrc->cmask_size) {
+		r600_blit_decompress_color(ctx, rsrc,
+					   0, 0,
+					   info->src.layer, info->src.layer);
+	}
 
 	/* this is correct for upside-down blits too */
 	u_box_2d(info->src.x0,
@@ -601,6 +676,10 @@ static void r600_resource_copy_region(struct pipe_context *ctx,
 					   src_box->z, src_box->z + src_box->depth - 1,
 					   0, u_max_sample(src));
 	}
+	if (rsrc->fmask_size && rsrc->cmask_size) {
+		r600_blit_decompress_color(ctx, rsrc, src_level, src_level,
+					   src_box->z, src_box->z + src_box->depth - 1);
+	}
 
 	restore_orig[0] = restore_orig[1] = FALSE;
 
diff --git a/src/gallium/drivers/r600/r600_pipe.c b/src/gallium/drivers/r600/r600_pipe.c
index 7936c475bb6..0d489100dfb 100644
--- a/src/gallium/drivers/r600/r600_pipe.c
+++ b/src/gallium/drivers/r600/r600_pipe.c
@@ -168,6 +168,9 @@ static void r600_destroy_context(struct pipe_context *context)
 	if (rctx->custom_blend_resolve) {
 		rctx->context.delete_blend_state(&rctx->context, rctx->custom_blend_resolve);
 	}
+	if (rctx->custom_blend_decompress) {
+		rctx->context.delete_blend_state(&rctx->context, rctx->custom_blend_decompress);
+	}
 	util_unreference_framebuffer_state(&rctx->framebuffer);
 
 	r600_context_fini(rctx);
@@ -262,6 +265,7 @@ static struct pipe_context *r600_create_context(struct pipe_screen *screen, void
 			goto fail;
 		rctx->custom_dsa_flush = evergreen_create_db_flush_dsa(rctx);
 		rctx->custom_blend_resolve = evergreen_create_resolve_blend(rctx);
+		rctx->custom_blend_decompress = evergreen_create_decompress_blend(rctx);
 		rctx->has_vertex_cache = !(rctx->family == CHIP_CEDAR ||
 					   rctx->family == CHIP_PALM ||
 					   rctx->family == CHIP_SUMO ||
diff --git a/src/gallium/drivers/r600/r600_pipe.h b/src/gallium/drivers/r600/r600_pipe.h
index 0aff0f680d8..65725b6da62 100644
--- a/src/gallium/drivers/r600/r600_pipe.h
+++ b/src/gallium/drivers/r600/r600_pipe.h
@@ -268,6 +268,7 @@ struct r600_samplerview_state {
 	uint32_t			enabled_mask;
 	uint32_t			dirty_mask;
 	uint32_t			compressed_depthtex_mask; /* which textures are depth */
+	uint32_t			compressed_colortex_mask;
 };
 
 struct r600_textures_info {
@@ -327,12 +328,14 @@ struct r600_context {
 	unsigned			r6xx_num_clause_temp_gprs;
 	void				*custom_dsa_flush;
 	void				*custom_blend_resolve;
+	void				*custom_blend_decompress;
 
 	struct r600_screen		*screen;
 	struct radeon_winsys		*ws;
 	struct r600_pipe_state		*states[R600_PIPE_NSTATES];
 	struct r600_vertex_element	*vertex_elements;
 	struct pipe_framebuffer_state	framebuffer;
+	unsigned			compressed_cb_mask;
 	unsigned			compute_cb_target_mask;
 	unsigned			db_shader_control;
 	unsigned			pa_sc_line_stipple;
@@ -481,6 +484,7 @@ void evergreen_pipe_shader_vs(struct pipe_context *ctx, struct r600_pipe_shader
 void evergreen_fetch_shader(struct pipe_context *ctx, struct r600_vertex_element *ve);
 void *evergreen_create_db_flush_dsa(struct r600_context *rctx);
 void *evergreen_create_resolve_blend(struct r600_context *rctx);
+void *evergreen_create_decompress_blend(struct r600_context *rctx);
 void evergreen_polygon_offset_update(struct r600_context *rctx);
 boolean evergreen_is_format_supported(struct pipe_screen *screen,
 				      enum pipe_format format,
@@ -504,6 +508,8 @@ void r600_blit_decompress_depth(struct pipe_context *ctx,
 		unsigned first_sample, unsigned last_sample);
 void r600_decompress_depth_textures(struct r600_context *rctx,
 				    struct r600_samplerview_state *textures);
+void r600_decompress_color_textures(struct r600_context *rctx,
+				    struct r600_samplerview_state *textures);
 
 /* r600_buffer.c */
 bool r600_init_resource(struct r600_screen *rscreen,
diff --git a/src/gallium/drivers/r600/r600_resource.h b/src/gallium/drivers/r600/r600_resource.h
index 7beab0fa110..dbb958e6925 100644
--- a/src/gallium/drivers/r600/r600_resource.h
+++ b/src/gallium/drivers/r600/r600_resource.h
@@ -55,6 +55,11 @@ struct r600_texture {
 	struct r600_texture		*flushed_depth_texture;
 	boolean				is_flushing_texture;
 	struct radeon_surface		surface;
+
+	/* FMASK and CMASK can only be used with MSAA textures for now.
+	 * MSAA textures cannot have mipmaps. */
+	unsigned			fmask_offset, fmask_size, fmask_bank_height;
+	unsigned			cmask_offset, cmask_size, cmask_slice_tile_max;
 };
 
 #define R600_TEX_IS_TILED(tex, level) ((tex)->array_mode[level] != V_038000_ARRAY_LINEAR_GENERAL && (tex)->array_mode[level] != V_038000_ARRAY_LINEAR_ALIGNED)
@@ -74,12 +79,14 @@ struct r600_surface {
 	unsigned cb_color_base;
 	unsigned cb_color_view;
 	unsigned cb_color_size;		/* R600 only */
-	unsigned cb_color_frag;		/* R600 only */
-	unsigned cb_color_tile;		/* R600 only */
 	unsigned cb_color_dim;		/* EG only */
 	unsigned cb_color_pitch;	/* EG only */
 	unsigned cb_color_slice;	/* EG only */
 	unsigned cb_color_attrib;	/* EG only */
+	unsigned cb_color_fmask;	/* CB_COLORn_FMASK (EG) or CB_COLORn_FRAG (r600) */
+	unsigned cb_color_fmask_slice;	/* EG only */
+	unsigned cb_color_cmask;	/* CB_COLORn_CMASK (EG) or CB_COLORn_TILE (r600) */
+	unsigned cb_color_cmask_slice;	/* EG only */
 
 	/* DB registers. */
 	unsigned db_depth_info;		/* DB_Z_INFO (EG) or DB_DEPTH_INFO (r600) */
diff --git a/src/gallium/drivers/r600/r600_state.c b/src/gallium/drivers/r600/r600_state.c
index d06c25000f1..8f6b0cd988b 100644
--- a/src/gallium/drivers/r600/r600_state.c
+++ b/src/gallium/drivers/r600/r600_state.c
@@ -1426,9 +1426,9 @@ static void r600_set_framebuffer_state(struct pipe_context *ctx,
 		r600_pipe_state_add_reg(rstate, R_028080_CB_COLOR0_VIEW + i * 4,
 					surf->cb_color_view);
 		r600_pipe_state_add_reg_bo(rstate, R_0280E0_CB_COLOR0_FRAG + i * 4,
-					   surf->cb_color_frag, res, RADEON_USAGE_READWRITE);
+					   surf->cb_color_fmask, res, RADEON_USAGE_READWRITE);
 		r600_pipe_state_add_reg_bo(rstate, R_0280C0_CB_COLOR0_TILE + i * 4,
-					   surf->cb_color_tile, res, RADEON_USAGE_READWRITE);
+					   surf->cb_color_cmask, res, RADEON_USAGE_READWRITE);
 	}
 	/* set CB_COLOR1_INFO for possible dual-src blending */
 	if (i == 1) {
diff --git a/src/gallium/drivers/r600/r600_state_common.c b/src/gallium/drivers/r600/r600_state_common.c
index 5895fefc14d..77c3e20599e 100644
--- a/src/gallium/drivers/r600/r600_state_common.c
+++ b/src/gallium/drivers/r600/r600_state_common.c
@@ -628,6 +628,12 @@ void r600_set_sampler_views(struct pipe_context *pipe,
 				dst->views.compressed_depthtex_mask &= ~(1 << i);
 			}
 
+			if (rtex->cmask_size && rtex->fmask_size) {
+				dst->views.compressed_colortex_mask |= 1 << i;
+			} else {
+				dst->views.compressed_colortex_mask &= ~(1 << i);
+			}
+
 			/* Changing from array to non-arrays textures and vice
 			 * versa requires updating TEX_ARRAY_OVERRIDE on R6xx-R7xx. */
 			if (rctx->chip_class <= R700 &&
@@ -649,6 +655,7 @@ void r600_set_sampler_views(struct pipe_context *pipe,
 	dst->views.enabled_mask |= new_mask;
 	dst->views.dirty_mask |= new_mask;
 	dst->views.compressed_depthtex_mask &= dst->views.enabled_mask;
+	dst->views.compressed_colortex_mask &= dst->views.enabled_mask;
 
 	r600_sampler_views_dirty(rctx, &dst->views);
 }
@@ -1046,6 +1053,12 @@ static void r600_update_derived_state(struct r600_context *rctx)
 		if (rctx->ps_samplers.views.compressed_depthtex_mask) {
 			r600_decompress_depth_textures(rctx, &rctx->ps_samplers.views);
 		}
+		if (rctx->vs_samplers.views.compressed_colortex_mask) {
+			r600_decompress_color_textures(rctx, &rctx->vs_samplers.views);
+		}
+		if (rctx->ps_samplers.views.compressed_colortex_mask) {
+			r600_decompress_color_textures(rctx, &rctx->ps_samplers.views);
+		}
 	}
 
 	r600_shader_select(ctx, rctx->ps_shader, &ps_dirty);
@@ -1262,6 +1275,20 @@ void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *dinfo)
 
 		rtex->dirty_level_mask |= 1 << surf->u.tex.level;
 	}
+	if (rctx->compressed_cb_mask) {
+		struct pipe_surface *surf;
+		struct r600_texture *rtex;
+		unsigned mask = rctx->compressed_cb_mask;
+
+		do {
+			unsigned i = u_bit_scan(&mask);
+			surf = rctx->framebuffer.cbufs[i];
+			rtex = (struct r600_texture*)surf->texture;
+
+			rtex->dirty_level_mask |= 1 << surf->u.tex.level;
+
+		} while (mask);
+	}
 
 	pipe_resource_reference(&ib.buffer, NULL);
 }
diff --git a/src/gallium/drivers/r600/r600_texture.c b/src/gallium/drivers/r600/r600_texture.c
index 90a834ec634..acb77d5c4dd 100644
--- a/src/gallium/drivers/r600/r600_texture.c
+++ b/src/gallium/drivers/r600/r600_texture.c
@@ -252,6 +252,97 @@ static const struct u_resource_vtbl r600_texture_vtbl =
 	NULL				/* transfer_inline_write */
 };
 
+static void r600_texture_allocate_fmask(struct r600_screen *rscreen,
+					struct r600_texture *rtex)
+{
+	/* FMASK is allocated pretty much like an ordinary texture.
+	 * Here we use bpe in the units of bits, not bytes. */
+	struct radeon_surface fmask = rtex->surface;
+	unsigned nr_samples = rtex->resource.b.b.nr_samples;
+
+	switch (nr_samples) {
+	case 2:
+		/* This should be 8,1, but we should set nsamples > 1
+		 * for the allocator to treat it as a multisample surface.
+		 * Let's set 4,2 then. */
+	case 4:
+		fmask.bpe = 4;
+		fmask.nsamples = 2;
+		break;
+	case 8:
+		fmask.bpe = 8;
+		fmask.nsamples = 4;
+		break;
+	case 16:
+		fmask.bpe = 16;
+		fmask.nsamples = 4;
+		break;
+	default:
+		R600_ERR("Invalid sample count for FMASK allocation.\n");
+		return;
+	}
+
+	if (rscreen->chip_class >= EVERGREEN) {
+		fmask.bankh = nr_samples <= 4 ? 4 : 1;
+	}
+
+	if (rscreen->ws->surface_init(rscreen->ws, &fmask)) {
+		R600_ERR("Got error in surface_init while allocating FMASK.\n");
+		return;
+	}
+	assert(fmask.level[0].mode == RADEON_SURF_MODE_2D);
+
+	/* Reserve space for FMASK while converting bits back to bytes. */
+	rtex->fmask_bank_height = fmask.bankh;
+	rtex->fmask_offset = align(rtex->size, MAX2(256, fmask.bo_alignment));
+	rtex->fmask_size = (fmask.bo_size + 7) / 8;
+	rtex->size = rtex->fmask_offset + rtex->fmask_size;
+#if 0
+	printf("FMASK width=%u, height=%i, bits=%u, size=%u\n",
+	       fmask.npix_x, fmask.npix_y, fmask.bpe * fmask.nsamples, rtex->fmask_size);
+#endif
+}
+
+static void r600_texture_allocate_cmask(struct r600_screen *rscreen,
+					struct r600_texture *rtex)
+{
+	unsigned cmask_tile_width = 8;
+	unsigned cmask_tile_height = 8;
+	unsigned cmask_tile_elements = cmask_tile_width * cmask_tile_height;
+	unsigned element_bits = 4;
+	unsigned cmask_cache_bits = 1024;
+	unsigned num_pipes = rscreen->tiling_info.num_channels;
+	unsigned pipe_interleave_bytes = rscreen->tiling_info.group_bytes;
+
+	unsigned elements_per_macro_tile = (cmask_cache_bits / element_bits) * num_pipes;
+	unsigned pixels_per_macro_tile = elements_per_macro_tile * cmask_tile_elements;
+	unsigned sqrt_pixels_per_macro_tile = sqrt(pixels_per_macro_tile);
+	unsigned macro_tile_width = util_next_power_of_two(sqrt_pixels_per_macro_tile);
+	unsigned macro_tile_height = pixels_per_macro_tile / macro_tile_width;
+
+	unsigned pitch_elements = align(rtex->surface.npix_x, macro_tile_width);
+	unsigned height = align(rtex->surface.npix_y, macro_tile_height);
+
+	unsigned base_align = num_pipes * pipe_interleave_bytes;
+	unsigned slice_bytes =
+		((pitch_elements * height * element_bits + 7) / 8) / cmask_tile_elements;
+	unsigned size = rtex->surface.array_size * align(slice_bytes, base_align);
+
+	assert(macro_tile_width % 128 == 0);
+	assert(macro_tile_height % 128 == 0);
+
+	rtex->cmask_slice_tile_max = ((pitch_elements * height) / (128*128)) - 1;
+	rtex->cmask_offset = align(rtex->size, MAX2(256, base_align));
+	rtex->cmask_size = size;
+	rtex->size = rtex->cmask_offset + rtex->cmask_size;
+#if 0
+	printf("CMASK: macro tile width = %u, macro tile height = %u, "
+	       "pitch elements = %u, height = %u, slice tile max = %u\n",
+	       macro_tile_width, macro_tile_height, pitch_elements, height,
+	       rtex->cmask_slice_tile_max);
+#endif
+}
+
 static struct r600_texture *
 r600_texture_create_object(struct pipe_screen *screen,
 			   const struct pipe_resource *base,
@@ -287,6 +378,17 @@ r600_texture_create_object(struct pipe_screen *screen,
 		return NULL;
 	}
 
+	if (base->nr_samples > 1 && !rtex->is_depth && alloc_bo) {
+		r600_texture_allocate_fmask(rscreen, rtex);
+		r600_texture_allocate_cmask(rscreen, rtex);
+	}
+
+	if (!rtex->is_depth && base->nr_samples > 1 &&
+	    (!rtex->fmask_size || !rtex->cmask_size)) {
+		FREE(rtex);
+		return NULL;
+	}
+
 	/* Now create the backing buffer. */
 	if (!buf && alloc_bo) {
 		unsigned base_align = rtex->surface.bo_alignment;
@@ -301,6 +403,13 @@ r600_texture_create_object(struct pipe_screen *screen,
 		resource->cs_buf = rscreen->ws->buffer_get_cs_handle(buf);
 		resource->domains = RADEON_DOMAIN_GTT | RADEON_DOMAIN_VRAM;
 	}
+
+	if (rtex->cmask_size) {
+		/* Initialize the cmask to 0xCC (= compressed state). */
+		char *ptr = rscreen->ws->buffer_map(resource->cs_buf, NULL, PIPE_TRANSFER_WRITE);
+		memset(ptr + rtex->cmask_offset, 0xCC, rtex->cmask_size);
+		rscreen->ws->buffer_unmap(resource->cs_buf);
+	}
 	return rtex;
 }