st/mesa: do vertex and fragment color clamping in shaders

For ARB_color_buffer_float. Most hardware can't do it and st/mesa is
the perfect place for a fallback.
The exceptions are:
- r500 (vertex clamp only)
- nv50 (both)
- nvc0 (both)
- softpipe (both)

We also have to take into account that r300 can do CLAMPED vertex colors only,
while r600 can do UNCLAMPED vertex colors only. The difference can be expressed
with the two new CAPs.

13 files changed, 120 insertions, 46 deletions

diff --git a/src/mesa/state_tracker/st_atom_rasterizer.c b/src/mesa/state_tracker/st_atom_rasterizer.c
index f3d28e675f5..25799bf2bd5 100644
--- a/src/mesa/state_tracker/st_atom_rasterizer.c
+++ b/src/mesa/state_tracker/st_atom_rasterizer.c
@@ -112,7 +112,8 @@ static void update_raster_state( struct st_context *st )
       raster->light_twoside = 1;
    }
 
-   raster->clamp_vertex_color = ctx->Light._ClampVertexColor;
+   raster->clamp_vertex_color = !st->clamp_vert_color_in_shader &&
+                                ctx->Light._ClampVertexColor;
 
    /* _NEW_POLYGON
     */
@@ -255,7 +256,8 @@ static void update_raster_state( struct st_context *st )
       raster->scissor = 1;
 
    /* _NEW_FRAG_CLAMP */
-   raster->clamp_fragment_color = ctx->Color._ClampFragmentColor;
+   raster->clamp_fragment_color = !st->clamp_frag_color_in_shader &&
+                                  ctx->Color._ClampFragmentColor;
    raster->gl_rasterization_rules = 1;
 
    /* _NEW_RASTERIZER_DISCARD */
diff --git a/src/mesa/state_tracker/st_atom_shader.c b/src/mesa/state_tracker/st_atom_shader.c
index c311d043931..ae3491097b4 100644
--- a/src/mesa/state_tracker/st_atom_shader.c
+++ b/src/mesa/state_tracker/st_atom_shader.c
@@ -83,6 +83,10 @@ update_fp( struct st_context *st )
    memset(&key, 0, sizeof(key));
    key.st = st;
 
+   /* _NEW_FRAG_CLAMP */
+   key.clamp_color = st->clamp_frag_color_in_shader &&
+                     st->ctx->Color._ClampFragmentColor;
+
    st->fp_variant = st_get_fp_variant(st, stfp, &key);
 
    st_reference_fragprog(st, &st->fp, stfp);
@@ -141,6 +145,9 @@ update_vp( struct st_context *st )
                                 st->ctx->Polygon.FrontMode != GL_FILL ||
                                 st->ctx->Polygon.BackMode != GL_FILL));
 
+   key.clamp_color = st->clamp_vert_color_in_shader &&
+                     st->ctx->Light._ClampVertexColor;
+
    st->vp_variant = st_get_vp_variant(st, stvp, &key);
 
    st_reference_vertprog(st, &st->vp, stvp);
diff --git a/src/mesa/state_tracker/st_cb_bitmap.c b/src/mesa/state_tracker/st_cb_bitmap.c
index a9709680208..7c0254d8d22 100644
--- a/src/mesa/state_tracker/st_cb_bitmap.c
+++ b/src/mesa/state_tracker/st_cb_bitmap.c
@@ -450,6 +450,8 @@ draw_bitmap_quad(struct gl_context *ctx, GLint x, GLint y, GLfloat z,
    memset(&key, 0, sizeof(key));
    key.st = st;
    key.bitmap = GL_TRUE;
+   key.clamp_color = st->clamp_frag_color_in_shader &&
+                     st->ctx->Color._ClampFragmentColor;
 
    fpv = st_get_fp_variant(st, st->fp, &key);
 
diff --git a/src/mesa/state_tracker/st_cb_drawpixels.c b/src/mesa/state_tracker/st_cb_drawpixels.c
index 13c4f3369cc..386eed2909f 100644
--- a/src/mesa/state_tracker/st_cb_drawpixels.c
+++ b/src/mesa/state_tracker/st_cb_drawpixels.c
@@ -684,7 +684,8 @@ draw_textured_quad(struct gl_context *ctx, GLint x, GLint y, GLfloat z,
    {
       struct pipe_rasterizer_state rasterizer;
       memset(&rasterizer, 0, sizeof(rasterizer));
-      rasterizer.clamp_fragment_color = ctx->Color._ClampFragmentColor;
+      rasterizer.clamp_fragment_color = !st->clamp_frag_color_in_shader &&
+                                        ctx->Color._ClampFragmentColor;
       rasterizer.gl_rasterization_rules = 1;
       rasterizer.depth_clip = !ctx->Transform.DepthClamp;
       rasterizer.scissor = ctx->Scissor.Enabled;
@@ -1007,6 +1008,8 @@ get_color_fp_variant(struct st_context *st)
                        ctx->Pixel.AlphaBias != 0.0 ||
                        ctx->Pixel.AlphaScale != 1.0);
    key.pixelMaps = ctx->Pixel.MapColorFlag;
+   key.clamp_color = st->clamp_frag_color_in_shader &&
+                     st->ctx->Color._ClampFragmentColor;
 
    fpv = st_get_fp_variant(st, st->fp, &key);
 
diff --git a/src/mesa/state_tracker/st_context.c b/src/mesa/state_tracker/st_context.c
index dc1d33f1dae..a3fd4dbcb7f 100644
--- a/src/mesa/state_tracker/st_context.c
+++ b/src/mesa/state_tracker/st_context.c
@@ -76,6 +76,17 @@ void st_invalidate_state(struct gl_context * ctx, GLuint new_state)
 {
    struct st_context *st = st_context(ctx);
 
+   /* Replace _NEW_FRAG_CLAMP with ST_NEW_FRAGMENT_PROGRAM for the fallback. */
+   if (st->clamp_frag_color_in_shader && (new_state & _NEW_FRAG_CLAMP)) {
+      new_state &= ~_NEW_FRAG_CLAMP;
+      st->dirty.st |= ST_NEW_FRAGMENT_PROGRAM;
+   }
+
+   /* Update the vertex shader if ctx->Light._ClampVertexColor was changed. */
+   if (st->clamp_vert_color_in_shader && (new_state & _NEW_LIGHT)) {
+      st->dirty.st |= ST_NEW_VERTEX_PROGRAM;
+   }
+
    st->dirty.mesa |= new_state;
    st->dirty.st |= ST_NEW_MESA;
 
diff --git a/src/mesa/state_tracker/st_context.h b/src/mesa/state_tracker/st_context.h
index 9db50b3f4c8..da03719fdec 100644
--- a/src/mesa/state_tracker/st_context.h
+++ b/src/mesa/state_tracker/st_context.h
@@ -76,6 +76,8 @@ struct st_context
    struct draw_stage *selection_stage;  /**< For GL_SELECT rendermode */
    struct draw_stage *rastpos_stage;  /**< For glRasterPos */
    GLboolean sw_primitive_restart;
+   GLboolean clamp_frag_color_in_shader;
+   GLboolean clamp_vert_color_in_shader;
 
 
    /* On old libGL's for linux we need to invalidate the drawables
diff --git a/src/mesa/state_tracker/st_extensions.c b/src/mesa/state_tracker/st_extensions.c
index 49c874710bd..1f3b59fc67e 100644
--- a/src/mesa/state_tracker/st_extensions.c
+++ b/src/mesa/state_tracker/st_extensions.c
@@ -607,33 +607,16 @@ void st_init_extensions(struct st_context *st)
       ctx->Extensions.ARB_depth_clamp = GL_TRUE;
    }
 
-   /* This extension does not actually require support of floating point
-    * render targets, just clamping controls.
-    * Advertise this extension if either fragment color clamping is supported
-    * or no render targets having color values outside of the range [0, 1]
-    * are supported, in which case the fragment color clamping has no effect
-    * on rendering.
-    */
-   if (screen->get_param(screen, PIPE_CAP_FRAGMENT_COLOR_CLAMP_CONTROL) ||
-       (!screen->is_format_supported(screen, PIPE_FORMAT_R8G8B8A8_SNORM,
-                                     PIPE_TEXTURE_2D, 0,
-                                     PIPE_BIND_RENDER_TARGET) &&
-        !screen->is_format_supported(screen, PIPE_FORMAT_R16G16B16A16_SNORM,
-                                     PIPE_TEXTURE_2D, 0,
-                                     PIPE_BIND_RENDER_TARGET) &&
-        !screen->is_format_supported(screen, PIPE_FORMAT_R16G16B16A16_FLOAT,
-                                     PIPE_TEXTURE_2D, 0,
-                                     PIPE_BIND_RENDER_TARGET) &&
-        !screen->is_format_supported(screen, PIPE_FORMAT_R32G32B32A32_FLOAT,
-                                     PIPE_TEXTURE_2D, 0,
-                                     PIPE_BIND_RENDER_TARGET) &&
-        !screen->is_format_supported(screen, PIPE_FORMAT_R11G11B10_FLOAT,
-                                     PIPE_TEXTURE_2D, 0,
-                                     PIPE_BIND_RENDER_TARGET) &&
-        !screen->is_format_supported(screen, PIPE_FORMAT_R9G9B9E5_FLOAT,
-                                     PIPE_TEXTURE_2D, 0,
-                                     PIPE_BIND_RENDER_TARGET))) {
+   if (screen->get_param(screen, PIPE_CAP_VERTEX_COLOR_UNCLAMPED)) {
       ctx->Extensions.ARB_color_buffer_float = GL_TRUE;
+
+      if (!screen->get_param(screen, PIPE_CAP_VERTEX_COLOR_CLAMPED)) {
+         st->clamp_vert_color_in_shader = TRUE;
+      }
+
+      if (!screen->get_param(screen, PIPE_CAP_FRAGMENT_COLOR_CLAMPED)) {
+         st->clamp_frag_color_in_shader = TRUE;
+      }
    }
 
    if (screen->get_param(screen, PIPE_CAP_SHADER_STENCIL_EXPORT)) {
diff --git a/src/mesa/state_tracker/st_glsl_to_tgsi.cpp b/src/mesa/state_tracker/st_glsl_to_tgsi.cpp
index 92dffe25866..188e8d9d5c9 100644
--- a/src/mesa/state_tracker/st_glsl_to_tgsi.cpp
+++ b/src/mesa/state_tracker/st_glsl_to_tgsi.cpp
@@ -4055,7 +4055,7 @@ src_register(struct st_translate *t,
 static struct ureg_dst
 translate_dst(struct st_translate *t,
               const st_dst_reg *dst_reg,
-              bool saturate)
+              bool saturate, bool clamp_color)
 {
    struct ureg_dst dst = dst_register(t, 
                                       dst_reg->file,
@@ -4065,6 +4065,27 @@ translate_dst(struct st_translate *t,
    
    if (saturate)
       dst = ureg_saturate(dst);
+   else if (clamp_color && dst_reg->file == PROGRAM_OUTPUT) {
+      /* Clamp colors for ARB_color_buffer_float. */
+      switch (t->procType) {
+      case TGSI_PROCESSOR_VERTEX:
+         /* XXX if the geometry shader is present, this must be done there
+          * instead of here. */
+         if (dst_reg->index == VERT_RESULT_COL0 ||
+             dst_reg->index == VERT_RESULT_COL1 ||
+             dst_reg->index == VERT_RESULT_BFC0 ||
+             dst_reg->index == VERT_RESULT_BFC1) {
+            dst = ureg_saturate(dst);
+         }
+         break;
+
+      case TGSI_PROCESSOR_FRAGMENT:
+         if (dst_reg->index >= FRAG_RESULT_COLOR) {
+            dst = ureg_saturate(dst);
+         }
+         break;
+      }
+   }
 
    if (dst_reg->reladdr != NULL)
       dst = ureg_dst_indirect(dst, ureg_src(t->address[0]));
@@ -4134,7 +4155,8 @@ translate_tex_offset(struct st_translate *t,
 
 static void
 compile_tgsi_instruction(struct st_translate *t,
-                         const glsl_to_tgsi_instruction *inst)
+                         const glsl_to_tgsi_instruction *inst,
+                         bool clamp_dst_color_output)
 {
    struct ureg_program *ureg = t->ureg;
    GLuint i;
@@ -4151,7 +4173,8 @@ compile_tgsi_instruction(struct st_translate *t,
    if (num_dst) 
       dst[0] = translate_dst(t, 
                              &inst->dst,
-                             inst->saturate);
+                             inst->saturate,
+                             clamp_dst_color_output);
 
    for (i = 0; i < num_src; i++) 
       src[i] = translate_src(t, &inst->src[i]);
@@ -4457,7 +4480,8 @@ st_translate_program(
    const GLuint outputMapping[],
    const ubyte outputSemanticName[],
    const ubyte outputSemanticIndex[],
-   boolean passthrough_edgeflags)
+   boolean passthrough_edgeflags,
+   boolean clamp_color)
 {
    struct st_translate *t;
    unsigned i;
@@ -4697,7 +4721,8 @@ st_translate_program(
     */
    foreach_iter(exec_list_iterator, iter, program->instructions) {
       set_insn_start(t, ureg_get_instruction_number(ureg));
-      compile_tgsi_instruction(t, (glsl_to_tgsi_instruction *)iter.get());
+      compile_tgsi_instruction(t, (glsl_to_tgsi_instruction *)iter.get(),
+                               clamp_color);
 
       if (t->prevInstWrotePointSize && proginfo->Id) {
          /* The previous instruction wrote to the (fake) vertex point size
diff --git a/src/mesa/state_tracker/st_glsl_to_tgsi.h b/src/mesa/state_tracker/st_glsl_to_tgsi.h
index 1f71f33fd47..55d59d571bb 100644
--- a/src/mesa/state_tracker/st_glsl_to_tgsi.h
+++ b/src/mesa/state_tracker/st_glsl_to_tgsi.h
@@ -49,7 +49,8 @@ enum pipe_error st_translate_program(
    const GLuint outputMapping[],
    const ubyte outputSemanticName[],
    const ubyte outputSemanticIndex[],
-   boolean passthrough_edgeflags);
+   boolean passthrough_edgeflags,
+   boolean clamp_color);
 
 void free_glsl_to_tgsi_visitor(struct glsl_to_tgsi_visitor *v);
 void get_pixel_transfer_visitor(struct st_fragment_program *fp,
diff --git a/src/mesa/state_tracker/st_mesa_to_tgsi.c b/src/mesa/state_tracker/st_mesa_to_tgsi.c
index 0764234bd4b..fc77089e733 100644
--- a/src/mesa/state_tracker/st_mesa_to_tgsi.c
+++ b/src/mesa/state_tracker/st_mesa_to_tgsi.c
@@ -306,7 +306,8 @@ st_translate_texture_target( GLuint textarget,
 static struct ureg_dst
 translate_dst( struct st_translate *t,
                const struct prog_dst_register *DstReg,
-               boolean saturate )
+               boolean saturate,
+               boolean clamp_color)
 {
    struct ureg_dst dst = dst_register( t, 
                                        DstReg->File,
@@ -317,6 +318,27 @@ translate_dst( struct st_translate *t,
    
    if (saturate)
       dst = ureg_saturate( dst );
+   else if (clamp_color && DstReg->File == PROGRAM_OUTPUT) {
+      /* Clamp colors for ARB_color_buffer_float. */
+      switch (t->procType) {
+      case TGSI_PROCESSOR_VERTEX:
+         /* XXX if the geometry shader is present, this must be done there
+          * instead of here. */
+         if (DstReg->Index == VERT_RESULT_COL0 ||
+             DstReg->Index == VERT_RESULT_COL1 ||
+             DstReg->Index == VERT_RESULT_BFC0 ||
+             DstReg->Index == VERT_RESULT_BFC1) {
+            dst = ureg_saturate(dst);
+         }
+         break;
+
+      case TGSI_PROCESSOR_FRAGMENT:
+         if (DstReg->Index >= FRAG_RESULT_COLOR) {
+            dst = ureg_saturate(dst);
+         }
+         break;
+      }
+   }
 
    if (DstReg->RelAddr)
       dst = ureg_dst_indirect( dst, ureg_src(t->address[0]) );
@@ -662,7 +684,8 @@ translate_opcode( unsigned op )
 static void
 compile_instruction(
    struct st_translate *t,
-   const struct prog_instruction *inst )
+   const struct prog_instruction *inst,
+   boolean clamp_dst_color_output)
 {
    struct ureg_program *ureg = t->ureg;
    GLuint i;
@@ -677,7 +700,8 @@ compile_instruction(
    if (num_dst) 
       dst[0] = translate_dst( t, 
                               &inst->DstReg,
-                              inst->SaturateMode );
+                              inst->SaturateMode,
+                              clamp_dst_color_output);
 
    for (i = 0; i < num_src; i++) 
       src[i] = translate_src( t, &inst->SrcReg[i] );
@@ -1016,7 +1040,8 @@ st_translate_mesa_program(
    const GLuint outputMapping[],
    const ubyte outputSemanticName[],
    const ubyte outputSemanticIndex[],
-   boolean passthrough_edgeflags )
+   boolean passthrough_edgeflags,
+   boolean clamp_color)
 {
    struct st_translate translate, *t;
    unsigned i;
@@ -1233,7 +1258,7 @@ st_translate_mesa_program(
     */
    for (i = 0; i < program->NumInstructions; i++) {
       set_insn_start( t, ureg_get_instruction_number( ureg ));
-      compile_instruction( t, &program->Instructions[i] );
+      compile_instruction( t, &program->Instructions[i], clamp_color );
 
       if (t->prevInstWrotePointSize && program->Id) {
          /* The previous instruction wrote to the (fake) vertex point size
diff --git a/src/mesa/state_tracker/st_mesa_to_tgsi.h b/src/mesa/state_tracker/st_mesa_to_tgsi.h
index 7563c8050e3..7e1a5abdc60 100644
--- a/src/mesa/state_tracker/st_mesa_to_tgsi.h
+++ b/src/mesa/state_tracker/st_mesa_to_tgsi.h
@@ -59,7 +59,8 @@ st_translate_mesa_program(
    const GLuint outputMapping[],
    const ubyte outputSemanticName[],
    const ubyte outputSemanticIndex[],
-   boolean passthrough_edgeflags );
+   boolean passthrough_edgeflags,
+   boolean clamp_color);
 
 void
 st_free_tokens(const struct tgsi_token *tokens);
diff --git a/src/mesa/state_tracker/st_program.c b/src/mesa/state_tracker/st_program.c
index 8d08b2b0f31..3ae79ddbf4b 100644
--- a/src/mesa/state_tracker/st_program.c
+++ b/src/mesa/state_tracker/st_program.c
@@ -351,7 +351,8 @@ st_translate_vertex_program(struct st_context *st,
                                    stvp->result_to_output,
                                    stvp->output_semantic_name,
                                    stvp->output_semantic_index,
-                                   key->passthrough_edgeflags );
+                                   key->passthrough_edgeflags,
+                                   key->clamp_color);
    else
       error = st_translate_mesa_program(st->ctx,
                                         TGSI_PROCESSOR_VERTEX,
@@ -368,7 +369,8 @@ st_translate_vertex_program(struct st_context *st,
                                         stvp->result_to_output,
                                         stvp->output_semantic_name,
                                         stvp->output_semantic_index,
-                                        key->passthrough_edgeflags );
+                                        key->passthrough_edgeflags,
+                                        key->clamp_color);
 
    if (error)
       goto fail;
@@ -505,7 +507,8 @@ st_translate_fragment_program(struct st_context *st,
    }
 #endif
 
-   if (!stfp->tgsi.tokens) {
+   /* XXX this will be cleaned up in the following commit */
+   if (1) {
       /* need to translate Mesa instructions to TGSI now */
       GLuint outputMapping[FRAG_RESULT_MAX];
       GLuint inputMapping[FRAG_ATTRIB_MAX];
@@ -718,7 +721,8 @@ st_translate_fragment_program(struct st_context *st,
                               fs_num_outputs,
                               outputMapping,
                               fs_output_semantic_name,
-                              fs_output_semantic_index, FALSE );
+                              fs_output_semantic_index, FALSE,
+                              key->clamp_color );
       else
          st_translate_mesa_program(st->ctx,
                                    TGSI_PROCESSOR_FRAGMENT,
@@ -734,7 +738,8 @@ st_translate_fragment_program(struct st_context *st,
                                    fs_num_outputs,
                                    outputMapping,
                                    fs_output_semantic_name,
-                                   fs_output_semantic_index, FALSE );
+                                   fs_output_semantic_index, FALSE,
+                                   key->clamp_color);
 
       stfp->tgsi.tokens = ureg_get_tokens( ureg, NULL );
       ureg_destroy( ureg );
@@ -1022,6 +1027,7 @@ st_translate_geometry_program(struct st_context *st,
                              outputMapping,
                              gs_output_semantic_name,
                              gs_output_semantic_index,
+                             FALSE,
                              FALSE);
 
    stgp->num_inputs = gs_num_inputs;
diff --git a/src/mesa/state_tracker/st_program.h b/src/mesa/state_tracker/st_program.h
index 699b6e8ccb7..0ae3420ecb4 100644
--- a/src/mesa/state_tracker/st_program.h
+++ b/src/mesa/state_tracker/st_program.h
@@ -55,6 +55,9 @@ struct st_fp_variant_key
    GLuint pixelMaps:1;            /**< glDrawPixels w/ pixel lookup map? */
    GLuint drawpixels_z:1;         /**< glDrawPixels(GL_DEPTH) */
    GLuint drawpixels_stencil:1;   /**< glDrawPixels(GL_STENCIL) */
+
+   /** for ARB_color_buffer_float */
+   GLuint clamp_color:1;
 };
 
 
@@ -98,6 +101,9 @@ struct st_vp_variant_key
 {
    struct st_context *st;          /**< variants are per-context */
    boolean passthrough_edgeflags;
+
+   /** for ARB_color_buffer_float */
+   boolean clamp_color;
 };