Fix mem leak in SSE code generation path (Michel Dänzer) and don't crash if _mesa_exec_malloc() returns NULL.

(picked from mesa_7_0_branch)

3 files changed, 31 insertions, 13 deletions

diff --git a/src/mesa/tnl/t_vertex_sse.c b/src/mesa/tnl/t_vertex_sse.c
index ad4cc62d5f7..9515d9f81f5 100644
--- a/src/mesa/tnl/t_vertex_sse.c
+++ b/src/mesa/tnl/t_vertex_sse.c
@@ -39,6 +39,12 @@
 #include "x86/common_x86_asm.h"
 
 
+/**
+ * Number of bytes to allocate for generated SSE functions
+ */
+#define MAX_SSE_CODE_SIZE 1024
+
+
 #define X    0
 #define Y    1
 #define Z    2
@@ -348,8 +354,6 @@ static GLboolean build_vertex_emit( struct x86_program *p )
    struct x86_reg vp1 = x86_make_reg(file_XMM, 2);
    GLubyte *fixup, *label;
 
-   x86_init_func(&p->func);
-   
    /* Push a few regs?
     */
    x86_push(&p->func, countEBP);
@@ -621,7 +625,10 @@ static GLboolean build_vertex_emit( struct x86_program *p )
    x86_pop(&p->func, countEBP);
    x86_ret(&p->func);
 
+   assert(!vtx->emit);
    vtx->emit = (tnl_emit_func)x86_get_func(&p->func);
+
+   assert( (char *) p->func.csr - (char *) p->func.store <= MAX_SSE_CODE_SIZE );
    return GL_TRUE;
 }
 
@@ -646,7 +653,10 @@ void _tnl_generate_sse_emit( GLcontext *ctx )
    p.identity = x86_make_reg(file_XMM, 6);
    p.chan0 = x86_make_reg(file_XMM, 7);
 
-   x86_init_func(&p.func);
+   if (!x86_init_func(&p.func, MAX_SSE_CODE_SIZE)) {
+      vtx->emit = NULL;
+      return;
+   }
 
    if (build_vertex_emit(&p)) {
       _tnl_register_fastpath( vtx, GL_TRUE );
diff --git a/src/mesa/x86/rtasm/x86sse.c b/src/mesa/x86/rtasm/x86sse.c
index 3ea37bb5e76..612cd51a6ee 100644
--- a/src/mesa/x86/rtasm/x86sse.c
+++ b/src/mesa/x86/rtasm/x86sse.c
@@ -1063,20 +1063,29 @@ struct x86_reg x86_fn_arg( struct x86_function *p,
 }
 
 
-void x86_init_func( struct x86_function *p )
-{
-   x86_init_func_size(p, 1024);
-}
-
-void x86_init_func_size( struct x86_function *p, GLuint code_size )
+/**
+ * Initialize an x86_function object, allocating space for up to
+ * 'code_size' bytes of code.
+ */
+GLboolean x86_init_func( struct x86_function *p, GLuint code_size )
 {
+   assert(!p->store);
    p->store = _mesa_exec_malloc(code_size);
-   p->csr = p->store;
+   if (p->store) {
+      p->csr = p->store;
+      return GL_TRUE;
+   }
+   else {
+      p->csr = NULL;
+      return GL_FALSE;
+   }
 }
 
 void x86_release_func( struct x86_function *p )
 {
-   _mesa_exec_free(p->store);
+   if (p->store)
+      _mesa_exec_free(p->store);
+   p->store = p->csr = NULL;
 }
 
 
diff --git a/src/mesa/x86/rtasm/x86sse.h b/src/mesa/x86/rtasm/x86sse.h
index 66fb852ac98..42b09937bca 100644
--- a/src/mesa/x86/rtasm/x86sse.h
+++ b/src/mesa/x86/rtasm/x86sse.h
@@ -80,8 +80,7 @@ enum sse_cc {
  */
 
 
-void x86_init_func( struct x86_function *p );
-void x86_init_func_size( struct x86_function *p, GLuint code_size );
+GLboolean x86_init_func( struct x86_function *p, GLuint code_size );
 void x86_release_func( struct x86_function *p );
 void (*x86_get_func( struct x86_function *p ))( void );