ir_to_mesa: Let check_resources halt compilation

Previously check_resources could fail, but we'd still try to optimize
the shader, do device-specific code generation, etc.  In some cases,
this could explode (especially in the device-specific code
generation).  I haven't found that I could trigger this with the
current code.  When too many samplers were used with the new uniform
handling code, I observed several crashes deep down in the driver.

NOTE: This is candidate for the 7.11 branch.

Signed-off-by: Ian Romanick <[email protected]>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=41609
Cc: Eric Anholt <[email protected]>
Reviewed-and-tested-by: Kenneth Graunke <[email protected]>

1 files changed, 17 insertions, 6 deletions

diff --git a/src/mesa/program/ir_to_mesa.cpp b/src/mesa/program/ir_to_mesa.cpp
index 93563bdaa5b..3bf2cc725b9 100644
--- a/src/mesa/program/ir_to_mesa.cpp
+++ b/src/mesa/program/ir_to_mesa.cpp
@@ -2546,7 +2546,7 @@ count_resources(struct gl_program *prog)
  *
  * XXX more checks are needed...
  */
-static void
+static bool
 check_resources(const struct gl_context *ctx,
                 struct gl_shader_program *shader_program,
                 struct gl_program *prog)
@@ -2586,6 +2586,8 @@ check_resources(const struct gl_context *ctx,
    default:
       _mesa_problem(ctx, "unexpected program type in check_resources()");
    }
+
+   return shader_program->LinkStatus;
 }
 
 class add_uniform_to_shader : public uniform_field_visitor {
@@ -3168,9 +3170,7 @@ get_mesa_program(struct gl_context *ctx,
    }
 
    if (!shader_program->LinkStatus) {
-      free(mesa_instructions);
-      _mesa_reference_program(ctx, &shader->Program, NULL);
-      return NULL;
+      goto fail_exit;
    }
 
    set_branchtargets(&v, mesa_instructions, num_instructions);
@@ -3191,10 +3191,16 @@ get_mesa_program(struct gl_context *ctx,
    prog->Instructions = mesa_instructions;
    prog->NumInstructions = num_instructions;
 
+   /* Setting this to NULL prevents a possible double free in the fail_exit
+    * path (far below).
+    */
+   mesa_instructions = NULL;
+
    do_set_program_inouts(shader->ir, prog, shader->Type == GL_FRAGMENT_SHADER);
    count_resources(prog);
 
-   check_resources(ctx, shader_program, prog);
+   if (!check_resources(ctx, shader_program, prog))
+      goto fail_exit;
 
    _mesa_reference_program(ctx, &shader->Program, prog);
 
@@ -3203,6 +3209,11 @@ get_mesa_program(struct gl_context *ctx,
    }
 
    return prog;
+
+fail_exit:
+   free(mesa_instructions);
+   _mesa_reference_program(ctx, &shader->Program, NULL);
+   return NULL;
 }
 
 extern "C" {
@@ -3301,7 +3312,7 @@ _mesa_ir_link_shader(struct gl_context *ctx, struct gl_shader_program *prog)
       _mesa_reference_program(ctx, &linked_prog, NULL);
    }
 
-   return GL_TRUE;
+   return prog->LinkStatus;
 }