gallivm: don't clamp reference value for shadow comparison for float formats

This is wrong both for OpenGL and d3d. (In fact clamping is a side effect
of converting to depth format, so this should really do quantization too
at least in d3d10 for the comparisons to be truly correct.)

Reviewed-by: Zack Rusin <[email protected]>

1 files changed, 17 insertions, 4 deletions

diff --git a/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c b/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c
index 6780d3e53c4..337b6f73fd1 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c
@@ -1445,6 +1445,8 @@ lp_build_sample_compare(struct lp_build_sample_context *bld,
    LLVMBuilderRef builder = bld->gallivm->builder;
    LLVMValueRef res, p;
    const unsigned chan = 0;
+   unsigned chan_type;
+   const struct util_format_description *format_desc;
 
    if (bld->static_sampler_state->compare_mode == PIPE_TEX_COMPARE_NONE)
       return;
@@ -1466,11 +1468,22 @@ lp_build_sample_compare(struct lp_build_sample_context *bld,
                       coord, tex);
    }
 
-   /* Clamp p coords to [0,1] */
-   p = lp_build_clamp(&bld->coord_bld, p,
-                      bld->coord_bld.zero,
-                      bld->coord_bld.one);
+   /* Clamp p coords to [0,1] for fixed function depth texture format */
+   format_desc = util_format_description(bld->static_texture_state->format);
+   /* not entirely sure we couldn't end up with non-valid swizzle here */
+   chan_type = format_desc->swizzle[0] <= UTIL_FORMAT_SWIZZLE_W ?
+                  format_desc->channel[format_desc->swizzle[0]].type :
+                  UTIL_FORMAT_TYPE_FLOAT;
+   if (chan_type != UTIL_FORMAT_TYPE_FLOAT) {
+      p = lp_build_clamp(&bld->coord_bld, p,
+                         bld->coord_bld.zero, bld->coord_bld.one);
+   }
 
+   /*
+    * technically this is not entirely correct for unorm depth as the ref value
+    * should be converted to the depth format (quantization!) and comparison
+    * then done in texture format.
+    */
    /* result = (p FUNC texel) ? 1 : 0 */
    res = lp_build_cmp(texel_bld, bld->static_sampler_state->compare_func,
                       p, texel[chan]);