draw: make sure the buffer is big enough to fit everything emitted by the gs

2 files changed, 27 insertions, 48 deletions

diff --git a/src/gallium/auxiliary/draw/draw_gs.c b/src/gallium/auxiliary/draw/draw_gs.c
index 145a062e2e8..15d4c5c15d2 100644
--- a/src/gallium/auxiliary/draw/draw_gs.c
+++ b/src/gallium/auxiliary/draw/draw_gs.c
@@ -37,6 +37,7 @@
 
 #include "util/u_math.h"
 #include "util/u_memory.h"
+#include "util/u_prim.h"
 
 #define MAX_PRIM_VERTICES 6
 /* fixme: move it from here */
@@ -154,35 +155,6 @@ void draw_delete_geometry_shader(struct draw_context *draw,
    FREE(dgs);
 }
 
-static INLINE int num_vertices_for_prim(int prim)
-{
-   switch(prim) {
-   case PIPE_PRIM_POINTS:
-      return 1;
-   case PIPE_PRIM_LINES:
-      return 2;
-   case PIPE_PRIM_LINE_LOOP:
-      return 2;
-   case PIPE_PRIM_LINE_STRIP:
-      return 2;
-   case PIPE_PRIM_TRIANGLES:
-      return 3;
-   case PIPE_PRIM_TRIANGLE_STRIP:
-      return 3;
-   case PIPE_PRIM_TRIANGLE_FAN:
-      return 3;
-   case PIPE_PRIM_LINES_ADJACENCY:
-   case PIPE_PRIM_LINE_STRIP_ADJACENCY:
-      return 4;
-   case PIPE_PRIM_TRIANGLES_ADJACENCY:
-   case PIPE_PRIM_TRIANGLE_STRIP_ADJACENCY:
-      return 6;
-   default:
-      assert(!"Bad geometry shader input");
-      return 0;
-   }
-}
-
 static void draw_fetch_geometry_input(struct draw_geometry_shader *shader,
                                       int start_primitive,
                                       int num_primitives,
@@ -192,7 +164,7 @@ static void draw_fetch_geometry_input(struct draw_geometry_shader *shader,
 {
    struct tgsi_exec_machine *machine = shader->machine;
    unsigned slot, vs_slot, k, j;
-   unsigned num_vertices = num_vertices_for_prim(shader->input_primitive);
+   unsigned num_vertices = u_vertices_per_prim(shader->input_primitive);
    int idx = 0;
 
    for (slot = 0, vs_slot = 0; slot < shader->info.num_inputs; slot++) {
@@ -299,10 +271,12 @@ int draw_geometry_shader_run(struct draw_geometry_shader *shader,
 {
    struct tgsi_exec_machine *machine = shader->machine;
    unsigned int i;
-   unsigned num_in_vertices = num_vertices_for_prim(shader->input_primitive);
+   unsigned num_in_vertices = u_vertices_per_prim(shader->input_primitive);
    unsigned num_in_primitives = count/num_in_vertices;
    unsigned inputs_from_vs = 0;
 
+   if (0) debug_printf("%s count = %d\n", __FUNCTION__, count);
+
    shader->emitted_vertices = 0;
    shader->emitted_primitives = 0;
 
diff --git a/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline.c b/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline.c
index afc146c6023..938b0b3c049 100644
--- a/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline.c
+++ b/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline.c
@@ -27,6 +27,7 @@
 
 #include "util/u_math.h"
 #include "util/u_memory.h"
+#include "util/u_prim.h"
 #include "draw/draw_context.h"
 #include "draw/draw_vbuf.h"
 #include "draw/draw_vertex.h"
@@ -51,6 +52,24 @@ struct fetch_pipeline_middle_end {
    unsigned opt;
 };
 
+static int max_out_vertex_count(
+   struct fetch_pipeline_middle_end *fpme, int count)
+{
+   struct draw_context *draw = fpme->draw;
+   unsigned alloc_count = align( count, 4 );
+
+   if (draw->gs.geometry_shader) {
+      unsigned input_primitives = count / u_vertices_per_prim(fpme->input_prim);
+      /* max GS output is number of input primitives * max output
+       * vertices per each invocation */
+      unsigned gs_max_verts = input_primitives *
+                              draw->gs.geometry_shader->max_output_vertices;
+      if (gs_max_verts > count)
+         alloc_count = align(gs_max_verts, 4);
+   }
+
+   return alloc_count;
+}
 
 static void fetch_pipeline_prepare( struct draw_pt_middle_end *middle,
                                     unsigned in_prim,
@@ -140,12 +159,7 @@ static void fetch_pipeline_run( struct draw_pt_middle_end *middle,
    struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
    unsigned opt = fpme->opt;
    struct vertex_header *pipeline_verts;
-   unsigned alloc_count = align( fetch_count, 4 );
-
-   if (draw->gs.geometry_shader &&
-       draw->gs.geometry_shader->max_output_vertices > fetch_count) {
-      alloc_count = align(draw->gs.geometry_shader->max_output_vertices, 4);
-   }
+   unsigned alloc_count = max_out_vertex_count(fpme, fetch_count);
 
    pipeline_verts =
       (struct vertex_header *)MALLOC(fpme->vertex_size * alloc_count);
@@ -236,11 +250,7 @@ static void fetch_pipeline_linear_run( struct draw_pt_middle_end *middle,
    struct draw_geometry_shader *geometry_shader = draw->gs.geometry_shader;
    unsigned opt = fpme->opt;
    struct vertex_header *pipeline_verts;
-   unsigned alloc_count = align( count, 4 );
-
-   if (geometry_shader && geometry_shader->max_output_vertices > count) {
-      alloc_count = align(geometry_shader->max_output_vertices, 4);
-   }
+   unsigned alloc_count = max_out_vertex_count(fpme, count);
 
    pipeline_verts =
       (struct vertex_header *)MALLOC(fpme->vertex_size * alloc_count);
@@ -329,12 +339,7 @@ static boolean fetch_pipeline_linear_run_elts( struct draw_pt_middle_end *middle
    struct draw_geometry_shader *geometry_shader = draw->gs.geometry_shader;
    unsigned opt = fpme->opt;
    struct vertex_header *pipeline_verts;
-   unsigned alloc_count = align( count, 4 );
-
-   if (draw->gs.geometry_shader &&
-       draw->gs.geometry_shader->max_output_vertices > count) {
-      alloc_count = align(draw->gs.geometry_shader->max_output_vertices, 4);
-   }
+   unsigned alloc_count = max_out_vertex_count(fpme, count);
 
    pipeline_verts =
       (struct vertex_header *)MALLOC(fpme->vertex_size * alloc_count);