refactor bounds checking code

1 files changed, 81 insertions, 67 deletions

diff --git a/src/mesa/main/api_validate.c b/src/mesa/main/api_validate.c
index ab8c3e19bac..3e4fe46b1df 100644
--- a/src/mesa/main/api_validate.c
+++ b/src/mesa/main/api_validate.c
@@ -30,6 +30,55 @@
 #include "state.h"
 
 
+/**
+ * Find the max index in the given element/index buffer
+ */
+static GLuint
+max_buffer_index(GLcontext *ctx, GLuint count, GLenum type,
+                 const void *indices,
+                 struct gl_buffer_object *elementBuf)
+{
+   const GLubyte *map = NULL;
+   GLuint max = 0;
+   GLint i;
+
+   if (elementBuf->Name) {
+      /* elements are in a user-defined buffer object.  need to map it */
+      map = ctx->Driver.MapBuffer(ctx,
+                                  GL_ELEMENT_ARRAY_BUFFER_ARB,
+                                  GL_READ_ONLY,
+                                  elementBuf);
+      /* Actual address is the sum of pointers */
+      indices = (const GLvoid *) ADD_POINTERS(map, (const GLubyte *) indices);
+   }
+
+   if (type == GL_UNSIGNED_INT) {
+      for (i = 0; i < count; i++)
+         if (((GLuint *) indices)[i] > max)
+            max = ((GLuint *) indices)[i];
+   }
+   else if (type == GL_UNSIGNED_SHORT) {
+      for (i = 0; i < count; i++)
+         if (((GLushort *) indices)[i] > max)
+            max = ((GLushort *) indices)[i];
+   }
+   else {
+      ASSERT(type == GL_UNSIGNED_BYTE);
+      for (i = 0; i < count; i++)
+         if (((GLubyte *) indices)[i] > max)
+            max = ((GLubyte *) indices)[i];
+   }
+
+   if (map) {
+      ctx->Driver.UnmapBuffer(ctx,
+                              GL_ELEMENT_ARRAY_BUFFER_ARB,
+                              ctx->Array.ElementArrayBufferObj);
+   }
+
+   return max;
+}
+
+
 GLboolean
 _mesa_validate_DrawElements(GLcontext *ctx,
 			    GLenum mode, GLsizei count, GLenum type,
@@ -61,21 +110,15 @@ _mesa_validate_DrawElements(GLcontext *ctx,
 
    /* Always need vertex positions */
    if (!ctx->Array.ArrayObj->Vertex.Enabled
-       && !(ctx->VertexProgram._Enabled && ctx->Array.ArrayObj->VertexAttrib[0].Enabled))
+       && !(ctx->VertexProgram._Enabled
+            && ctx->Array.ArrayObj->VertexAttrib[0].Enabled))
       return GL_FALSE;
 
    /* Vertex buffer object tests */
    if (ctx->Array.ElementArrayBufferObj->Name) {
-      GLuint indexBytes;
-
       /* use indices in the buffer object */
-      if (!ctx->Array.ElementArrayBufferObj->Size) {
-         _mesa_warning(ctx,
-                       "glDrawElements called with empty array elements buffer");
-         return GL_FALSE;
-      }
+      GLuint indexBytes;
 
-      /* make sure count doesn't go outside buffer bounds */
       if (type == GL_UNSIGNED_INT) {
          indexBytes = count * sizeof(GLuint);
       }
@@ -87,6 +130,7 @@ _mesa_validate_DrawElements(GLcontext *ctx,
          indexBytes = count * sizeof(GLushort);
       }
 
+      /* make sure count doesn't go outside buffer bounds */
       if (indexBytes > ctx->Array.ElementArrayBufferObj->Size) {
          _mesa_warning(ctx, "glDrawElements index out of buffer bounds");
          return GL_FALSE;
@@ -100,39 +144,8 @@ _mesa_validate_DrawElements(GLcontext *ctx,
 
    if (ctx->Const.CheckArrayBounds) {
       /* find max array index */
-      const GLubyte *map;
-      GLuint max = 0;
-      GLint i;
-
-      map = ctx->Driver.MapBuffer(ctx,
-                                  GL_ELEMENT_ARRAY_BUFFER_ARB,
-                                  GL_READ_ONLY,
-                                  ctx->Array.ElementArrayBufferObj);
-
-      /* Actual address is the sum of pointers */
-      indices = (const GLvoid *) ADD_POINTERS(map, (const GLubyte *) indices);
-
-      if (type == GL_UNSIGNED_INT) {
-         for (i = 0; i < count; i++)
-            if (((GLuint *) indices)[i] > max)
-               max = ((GLuint *) indices)[i];
-      }
-      else if (type == GL_UNSIGNED_SHORT) {
-         for (i = 0; i < count; i++)
-            if (((GLushort *) indices)[i] > max)
-               max = ((GLushort *) indices)[i];
-      }
-      else {
-         ASSERT(type == GL_UNSIGNED_BYTE);
-         for (i = 0; i < count; i++)
-            if (((GLubyte *) indices)[i] > max)
-               max = ((GLubyte *) indices)[i];
-      }
-
-      ctx->Driver.UnmapBuffer(ctx,
-                              GL_ELEMENT_ARRAY_BUFFER_ARB,
-                              ctx->Array.ElementArrayBufferObj);
-
+      GLuint max = max_buffer_index(ctx, count, type, indices,
+                                    ctx->Array.ElementArrayBufferObj);
       if (max >= ctx->Array._MaxElement) {
          /* the max element is out of bounds of one or more enabled arrays */
          return GL_FALSE;
@@ -179,41 +192,41 @@ _mesa_validate_DrawRangeElements(GLcontext *ctx, GLenum mode,
 
    /* Always need vertex positions */
    if (!ctx->Array.ArrayObj->Vertex.Enabled
-       && !(ctx->VertexProgram._Enabled && ctx->Array.ArrayObj->VertexAttrib[0].Enabled))
+       && !(ctx->VertexProgram._Enabled
+            && ctx->Array.ArrayObj->VertexAttrib[0].Enabled))
       return GL_FALSE;
 
    /* Vertex buffer object tests */
    if (ctx->Array.ElementArrayBufferObj->Name) {
-      /* XXX re-use code from above? */
+      /* use indices in the buffer object */
+      GLuint indexBytes;
+
+      if (type == GL_UNSIGNED_INT) {
+         indexBytes = count * sizeof(GLuint);
+      }
+      else if (type == GL_UNSIGNED_BYTE) {
+         indexBytes = count * sizeof(GLubyte);
+      }
+      else {
+         ASSERT(type == GL_UNSIGNED_SHORT);
+         indexBytes = count * sizeof(GLushort);
+      }
+
+      /* make sure count doesn't go outside buffer bounds */
+      if (indexBytes > ctx->Array.ElementArrayBufferObj->Size) {
+         _mesa_warning(ctx, "glDrawElements index out of buffer bounds");
+         return GL_FALSE;
+      }
    }
    else {
-      /* not using VBO */
+      /* not using a VBO */
       if (!indices)
          return GL_FALSE;
    }
 
    if (ctx->Const.CheckArrayBounds) {
-      /* Find max array index.
-       * We don't trust the user's start and end values.
-       */
-      GLuint max = 0;
-      GLint i;
-      if (type == GL_UNSIGNED_INT) {
-         for (i = 0; i < count; i++)
-            if (((GLuint *) indices)[i] > max)
-               max = ((GLuint *) indices)[i];
-      }
-      else if (type == GL_UNSIGNED_SHORT) {
-         for (i = 0; i < count; i++)
-            if (((GLushort *) indices)[i] > max)
-               max = ((GLushort *) indices)[i];
-      }
-      else {
-         ASSERT(type == GL_UNSIGNED_BYTE);
-         for (i = 0; i < count; i++)
-            if (((GLubyte *) indices)[i] > max)
-               max = ((GLubyte *) indices)[i];
-      }
+      GLuint max = max_buffer_index(ctx, count, type, indices,
+                                    ctx->Array.ElementArrayBufferObj);
       if (max >= ctx->Array._MaxElement) {
          /* the max element is out of bounds of one or more enabled arrays */
          return GL_FALSE;
@@ -248,7 +261,8 @@ _mesa_validate_DrawArrays(GLcontext *ctx,
       _mesa_update_state(ctx);
 
    /* Always need vertex positions */
-   if (!ctx->Array.ArrayObj->Vertex.Enabled && !ctx->Array.ArrayObj->VertexAttrib[0].Enabled)
+   if (!ctx->Array.ArrayObj->Vertex.Enabled
+       && !ctx->Array.ArrayObj->VertexAttrib[0].Enabled)
       return GL_FALSE;
 
    if (ctx->Const.CheckArrayBounds) {