draw: fix line stippling with aa lines

In contrast to non-aa, where stippling is based on either dx or dy
(depending on if it's a x or y major line), stippling is based on
actual distance with smooth lines, so adjust for this.

(It looks like there's some minor artifacts with mesa demos
line-sample and stippling, it looks like the line endpoints
aren't quite right with aa + stippling - maybe due to the
integer math in the stipple stage, but I can't quite pinpoint it.)

Reviewed-by: Brian Paul <[email protected]>
Reviewed-by: Jose Fonseca <[email protected]>

1 files changed, 13 insertions, 4 deletions

diff --git a/src/gallium/auxiliary/draw/draw_pipe_stipple.c b/src/gallium/auxiliary/draw/draw_pipe_stipple.c
index 3a84d6c3ea4..3a44e96add0 100644
--- a/src/gallium/auxiliary/draw/draw_pipe_stipple.c
+++ b/src/gallium/auxiliary/draw/draw_pipe_stipple.c
@@ -50,6 +50,7 @@ struct stipple_stage {
    float counter;
    uint pattern;
    uint factor;
+   bool smooth;
 };
 
 
@@ -133,12 +134,19 @@ stipple_line(struct draw_stage *stage, struct prim_header *header)
    float y0 = pos0[1];
    float y1 = pos1[1];
 
-   float dx = x0 > x1 ? x0 - x1 : x1 - x0;
-   float dy = y0 > y1 ? y0 - y1 : y1 - y0;
-
-   float length = MAX2(dx, dy);
+   float length;
    int i;
 
+   if (stipple->smooth) {
+      float dx = x1 - x0;
+      float dy = y1 - y0;
+      length = sqrtf(dx*dx + dy*dy);
+   } else {
+      float dx = x0 > x1 ? x0 - x1 : x1 - x0;
+      float dy = y0 > y1 ? y0 - y1 : y1 - y0;
+      length = MAX2(dx, dy);
+   }
+
    if (header->flags & DRAW_PIPE_RESET_STIPPLE)
       stipple->counter = 0;
 
@@ -205,6 +213,7 @@ stipple_first_line(struct draw_stage *stage,
 
    stipple->pattern = draw->rasterizer->line_stipple_pattern;
    stipple->factor = draw->rasterizer->line_stipple_factor + 1;
+   stipple->smooth = draw->rasterizer->line_smooth;
 
    stage->line = stipple_line;
    stage->line(stage, header);