draw/gs: reduce the size of the gs output buffer

We used to overallocate the output buffer sometimes running out
of memory with applications rendering large geometries. The actual
maximum number of vertices out is simply the maximum number of
primitives in (number of gs invocations) multiplied by the maximum
number of output vertices per gs input primitive (i.e. gs invocation).

Signed-off-by: Zack Rusin <[email protected]>
Reviewed-by: Jose Fonseca <[email protected]>
Reviewed-by: Roland Scheidegger <[email protected]>

1 files changed, 13 insertions, 7 deletions

diff --git a/src/gallium/auxiliary/draw/draw_gs.c b/src/gallium/auxiliary/draw/draw_gs.c
index 97e8a90f284..7de5e0308ec 100644
--- a/src/gallium/auxiliary/draw/draw_gs.c
+++ b/src/gallium/auxiliary/draw/draw_gs.c
@@ -552,6 +552,10 @@ int draw_geometry_shader_run(struct draw_geometry_shader *shader,
       u_decomposed_prims_for_vertices(shader->output_primitive,
                                       shader->max_output_vertices)
       * num_in_primitives;
+   /* we allocate exactly one extra vertex per primitive to allow the GS to emit
+    * overflown vertices into some area where they won't harm anyone */
+   unsigned total_verts_per_buffer = shader->primitive_boundary *
+      num_in_primitives;
 
    //Assume at least one primitive
    max_out_prims = MAX2(max_out_prims, 1);
@@ -559,23 +563,25 @@ int draw_geometry_shader_run(struct draw_geometry_shader *shader,
 
    output_verts->vertex_size = vertex_size;
    output_verts->stride = output_verts->vertex_size;
-   /* we allocate exactly one extra vertex per primitive to allow the GS to emit
-    * overflown vertices into some area where they won't harm anyone */
    output_verts->verts =
       (struct vertex_header *)MALLOC(output_verts->vertex_size *
-                                     max_out_prims *
-                                     shader->primitive_boundary);
+                                     total_verts_per_buffer);
+   debug_assert(output_verts->verts);
 
 #if 0
    debug_printf("%s count = %d (in prims # = %d)\n",
                 __FUNCTION__, num_input_verts, num_in_primitives);
    debug_printf("\tlinear = %d, prim_info->count = %d\n",
                 input_prim->linear, input_prim->count);
-   debug_printf("\tprim pipe = %s, shader in = %s, shader out = %s, max out = %d\n",
+   debug_printf("\tprim pipe = %s, shader in = %s, shader out = %s\n"
                 u_prim_name(input_prim->prim),
                 u_prim_name(shader->input_primitive),
-                u_prim_name(shader->output_primitive),
-                shader->max_output_vertices);
+                u_prim_name(shader->output_primitive));
+   debug_printf("\tmaxv  = %d, maxp = %d, primitive_boundary = %d, "
+                "vertex_size = %d, tverts = %d\n",
+                shader->max_output_vertices, max_out_prims,
+                shader->primitive_boundary, output_verts->vertex_size,
+                total_verts_per_buffer);
 #endif
 
    shader->emitted_vertices = 0;