swrast: fix broken _swrast_depth_clamp_span()

The integer Z clamping range depends on the number of bits
in the Z buffer because that's the scale factor used when we
transform NDC coords by the viewport/depth range.

Fixes fd.o bug #25972 but only for Z buffers up to a depth
of 30 bits.  Beyond that we get into messy integer overflow
issues and things fall apart.

1 files changed, 27 insertions, 14 deletions

diff --git a/src/mesa/swrast/s_depth.c b/src/mesa/swrast/s_depth.c
index 0b6bb7e3ec9..ac5dae2148b 100644
--- a/src/mesa/swrast/s_depth.c
+++ b/src/mesa/swrast/s_depth.c
@@ -497,17 +497,24 @@ depth_test_span32( GLcontext *ctx, GLuint n,
    return passed;
 }
 
-/* Apply ARB_depth_clamp to span of fragments. */
+
+
+/**
+ * Clamp fragment Z values to the depth near/far range (glDepthRange()).
+ * This is used when GL_ARB_depth_clamp/GL_DEPTH_CLAMP is turned on.
+ * In that case, vertexes are not clipped against the near/far planes
+ * so rasterization will produce fragment Z values outside the usual
+ * [0,1] range.
+ */
 void
 _swrast_depth_clamp_span( GLcontext *ctx, SWspan *span )
 {
    struct gl_framebuffer *fb = ctx->DrawBuffer;
-   struct gl_renderbuffer *rb = fb->_DepthBuffer;
    const GLuint count = span->end;
-   GLuint *zValues = span->array->z;
-   GLuint min, max;
+   GLint *zValues = (GLint *) span->array->z; /* sign change */
+   GLint min, max;
    GLfloat min_f, max_f;
-   int i;
+   GLuint i;
 
    if (ctx->Viewport.Near < ctx->Viewport.Far) {
       min_f = ctx->Viewport.Near;
@@ -517,15 +524,21 @@ _swrast_depth_clamp_span( GLcontext *ctx, SWspan *span )
       max_f = ctx->Viewport.Near;
    }
 
-   if (rb->DataType == GL_UNSIGNED_SHORT) {
-      CLAMPED_FLOAT_TO_USHORT(min, min_f);
-      CLAMPED_FLOAT_TO_USHORT(max, max_f);
-   } else {
-      assert(rb->DataType == GL_UNSIGNED_INT);
-      min = FLOAT_TO_UINT(min_f);
-      max = FLOAT_TO_UINT(max_f);
-   }
-
+   /* Convert floating point values in [0,1] to device Z coordinates in
+    * [0, DepthMax].
+    * ex: If the the Z buffer has 24 bits, DepthMax = 0xffffff.
+    * 
+    * XXX this all falls apart if we have 31 or more bits of Z because
+    * the triangle rasterization code produces unsigned Z values.  Negative
+    * vertex Z values come out as large fragment Z uints.
+    */
+   min = (GLint) (min_f * fb->_DepthMaxF);
+   max = (GLint) (max_f * fb->_DepthMaxF);
+   if (max < 0)
+      max = 0x7fffffff; /* catch over flow for 30-bit z */
+
+   /* Note that we do the comparisons here using signed integers.
+    */
    for (i = 0; i < count; i++) {
       if (zValues[i] < min)
 	 zValues[i] = min;