This should fix the flickering line bugs with deinterlace and decomb. Yadif needs the edges mirrored, and decomb's cubic interpolation has to be disabled at the tops and bottoms of frames.

git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@1862 b64f7644-9d1e-0410-96f1-a4d463321fa5

2 files changed, 120 insertions, 25 deletions

diff --git a/libhb/decomb.c b/libhb/decomb.c
index df1838a7d..e83f0d5a6 100644
--- a/libhb/decomb.c
+++ b/libhb/decomb.c
@@ -630,6 +630,13 @@ static void yadif_filter_line( uint8_t *dst,
     int refs = pv->ref_stride[plane];
     int x;
     
+    /* Decomb's cubic interpolation can only function when there are
+       three samples above and below, so regress to yadif's traditional
+       two-tap interpolation when filtering at the top and bottom edges. */
+    int edge = 0;
+    if( ( y < 3 ) || ( y > ( pv->height[plane] - 4 ) )  )
+        edge = 1;
+
     for( x = 0; x < w; x++)
     {
         /* Pixel above*/
@@ -654,7 +661,7 @@ static void yadif_filter_line( uint8_t *dst,
         int spatial_pred;
          
         /* Spatial pred is either a bilinear or cubic vertical interpolation. */
-        if( pv->mode > 0 )
+        if( pv->mode > 0 && !edge)
         {
             spatial_pred = cubic_interpolate( cur[-3*refs], cur[-refs], cur[+refs], cur[3*refs] );
         }
@@ -675,7 +682,7 @@ static void yadif_filter_line( uint8_t *dst,
                       + ABS(cur[-refs+1+j] - cur[+refs+1-j]);\
             if( score < spatial_score ){\
                 spatial_score = score;\
-                if( pv->mode > 0 )\
+                if( pv->mode > 0 && !edge )\
                 {\
                     switch(j)\
                     {\
@@ -748,7 +755,7 @@ void yadif_decomb_filter_thread( void *thread_args_v )
     int segment, segment_start, segment_stop;
     yadif_thread_arg_t *thread_args = thread_args_v;
     uint8_t **dst;
-    int parity, tff, y, w, h, ref_stride, is_combed;
+    int parity, tff, y, w, h, penultimate, ultimate, ref_stride, is_combed;
 
     pv = thread_args->pv;
     segment = thread_args->segment;
@@ -794,6 +801,8 @@ void yadif_decomb_filter_thread( void *thread_args_v )
             tff = yadif_work->tff;
             w = pv->width[plane];
             h = pv->height[plane];
+            penultimate = h - 2;
+            ultimate = h - 1;
             ref_stride = pv->ref_stride[plane];
             segment_start = ( h / pv->cpu_count ) * segment;
             if( segment == pv->cpu_count - 1 )
@@ -810,6 +819,7 @@ void yadif_decomb_filter_thread( void *thread_args_v )
             {
                 if( ( pv->mode == 4 && is_combed ) || is_combed == 2 )
                 {
+                    /* This line gets blend filtered, not yadif filtered. */
                     uint8_t *prev = &pv->ref[0][plane][y*ref_stride];
                     uint8_t *cur  = &pv->ref[1][plane][y*ref_stride];
                     uint8_t *next = &pv->ref[2][plane][y*ref_stride];
@@ -817,14 +827,59 @@ void yadif_decomb_filter_thread( void *thread_args_v )
 
                     blend_filter_line( dst2, cur, plane, y, pv );
                 }
-                else if( (y ^ parity) & 1 && is_combed == 1 )
+                else if( ( ( y ^ parity ) &  1 )  && ( is_combed == 1 ) )
                 {
-                    uint8_t *prev = &pv->ref[0][plane][y*ref_stride];
-                    uint8_t *cur  = &pv->ref[1][plane][y*ref_stride];
-                    uint8_t *next = &pv->ref[2][plane][y*ref_stride];
-                    uint8_t *dst2 = &dst[plane][y*w];
-
-                    yadif_filter_line( dst2, prev, cur, next, plane, parity ^ tff, y, pv );
+                    /* This line gets yadif filtered. It is the bottom field
+                       when TFF and vice-versa. It's the field that gets
+                       filtered. Because yadif needs 2 lines above and below
+                       the one being filtered, we need to mirror the edges.
+                       When TFF, this means replacing the 2nd line with a
+                       copy of the 1st, and the last with the second-to-last. */
+                    if( y > 1 && y < ( h -2 ) )
+                    {
+                        /* This isn't the top or bottom, proceed as normal to yadif. */
+                        uint8_t *prev = &pv->ref[0][plane][y*ref_stride];
+                        uint8_t *cur  = &pv->ref[1][plane][y*ref_stride];
+                        uint8_t *next = &pv->ref[2][plane][y*ref_stride];
+                        uint8_t *dst2 = &dst[plane][y*w];
+
+                        yadif_filter_line( dst2, 
+                                           prev, 
+                                           cur, 
+                                           next, 
+                                           plane, 
+                                           parity ^ tff,
+                                           y, 
+                                           pv );
+                    }
+                    else if( y == 0 )
+                    {
+                        /* BFF, so y0 = y1 */
+                        memcpy( &dst[plane][y*w],
+                                &pv->ref[1][plane][1*ref_stride],
+                                w * sizeof(uint8_t) );
+                    }
+                    else if( y == 1 )
+                    {
+                        /* TFF, so y1 = y0 */
+                        memcpy( &dst[plane][y*w],
+                                &pv->ref[1][plane][0],
+                                w * sizeof(uint8_t) );
+                    }
+                    else if( y == penultimate )
+                    {
+                        /* BFF, so penultimate y = ultimate y */
+                        memcpy( &dst[plane][y*w],
+                                &pv->ref[1][plane][ultimate*ref_stride],
+                                w * sizeof(uint8_t) );
+                    }
+                    else if( y == ultimate )
+                    {
+                        /* TFF, so ultimate y = penultimate y */
+                        memcpy( &dst[plane][y*w],
+                                &pv->ref[1][plane][penultimate*ref_stride],
+                                w * sizeof(uint8_t) );
+                    }
                 }
                 else
                 {
diff --git a/libhb/deinterlace.c b/libhb/deinterlace.c
index 728363428..9e4c33eb4 100644
--- a/libhb/deinterlace.c
+++ b/libhb/deinterlace.c
@@ -216,7 +216,7 @@ void yadif_filter_thread( void *thread_args_v )
     int segment, segment_start, segment_stop;
     yadif_thread_arg_t *thread_args = thread_args_v;
     uint8_t **dst;
-    int parity, tff, y, w, h, ref_stride;
+    int parity, tff, y, w, h, ref_stride, penultimate, ultimate;
 
 
     pv = thread_args->pv;
@@ -261,6 +261,8 @@ void yadif_filter_thread( void *thread_args_v )
             tff = yadif_work->tff;
             w = pv->width[plane];
             h = pv->height[plane];
+            penultimate = h -2;
+            ultimate = h - 1;
             ref_stride = pv->yadif_ref_stride[plane];
             segment_start = ( h / pv->cpu_count ) * segment;
             if( segment == pv->cpu_count - 1 )
@@ -275,24 +277,62 @@ void yadif_filter_thread( void *thread_args_v )
 
             for( y = segment_start; y < segment_stop; y++ )
             {
-                if( (y ^ parity) &  1 )
+                if( ( ( y ^ parity ) &  1 ) )
                 {
-                    uint8_t *prev = &pv->yadif_ref[0][plane][y*ref_stride];
-                    uint8_t *cur  = &pv->yadif_ref[1][plane][y*ref_stride];
-                    uint8_t *next = &pv->yadif_ref[2][plane][y*ref_stride];
-                    uint8_t *dst2 = &dst[plane][y*w];
-                    
-                    yadif_filter_line( dst2, 
-                                       prev, 
-                                       cur, 
-                                       next, 
-                                       plane, 
-                                       parity ^ tff, 
-                                       pv );
-                    
+                    /* This is the bottom field when TFF and vice-versa.
+                       It's the field that gets filtered. Because yadif
+                       needs 2 lines above and below the one being filtered,
+                       we need to mirror the edges. When TFF, this means
+                       replacing the 2nd line with a copy of the 1st,
+                       and the last with the second-to-last.                  */
+                    if( y > 1 && y < ( h -2 ) )
+                    {
+                        /* This isn't the top or bottom, proceed as normal to yadif. */
+                        uint8_t *prev = &pv->yadif_ref[0][plane][y*ref_stride];
+                        uint8_t *cur  = &pv->yadif_ref[1][plane][y*ref_stride];
+                        uint8_t *next = &pv->yadif_ref[2][plane][y*ref_stride];
+                        uint8_t *dst2 = &dst[plane][y*w];
+
+                        yadif_filter_line( dst2, 
+                                           prev, 
+                                           cur, 
+                                           next, 
+                                           plane, 
+                                           parity ^ tff, 
+                                           pv );
+                    }
+                    else if( y == 0 )
+                    {
+                        /* BFF, so y0 = y1 */
+                        memcpy( &dst[plane][y*w],
+                                &pv->yadif_ref[1][plane][1*ref_stride],
+                                w * sizeof(uint8_t) );
+                    }
+                    else if( y == 1 )
+                    {
+                        /* TFF, so y1 = y0 */
+                        memcpy( &dst[plane][y*w],
+                                &pv->yadif_ref[1][plane][0],
+                                w * sizeof(uint8_t) );
+                    }
+                    else if( y == penultimate )
+                    {
+                        /* BFF, so penultimate y = ultimate y */
+                        memcpy( &dst[plane][y*w],
+                                &pv->yadif_ref[1][plane][ultimate*ref_stride],
+                                w * sizeof(uint8_t) );
+                    }
+                    else if( y == ultimate )
+                    {
+                        /* TFF, so ultimate y = penultimate y */
+                        memcpy( &dst[plane][y*w],
+                                &pv->yadif_ref[1][plane][penultimate*ref_stride],
+                                w * sizeof(uint8_t) );
+                    }
                 }
                 else
                 {
+                    /* Preserve this field unfiltered */
                     memcpy( &dst[plane][y*w],
                             &pv->yadif_ref[1][plane][y*ref_stride],
                             w * sizeof(uint8_t) );