libhb: fix comb detection crash and decomb crash

hb_detect_comb() could crash because the dimensions of the video buffer
don't have to match the dimensions returned by work object info() method
if the video has segments of differeing resolutions.

decomb was allocating reference buffers that were too small.  This bug appears
to have always existed but doesn't usually get triggered because malloc
usually rounds allocation sizes up.


git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@4812 b64f7644-9d1e-0410-96f1-a4d463321fa5

4 files changed, 20 insertions, 34 deletions

diff --git a/libhb/decomb.c b/libhb/decomb.c
index 40bcb2a6c..e9519700f 100644
--- a/libhb/decomb.c
+++ b/libhb/decomb.c
@@ -1997,8 +1997,8 @@ static int hb_decomb_init( hb_filter_object_t * filter,
     for( i = 0; i < 3; i++ )
     {
         int is_chroma = !!i;
-        int w = ((init->width   + 31) & (~31))>>is_chroma;
-        int h = ((init->height+6+ 31) & (~31))>>is_chroma;
+        int w = ((init->width >>is_chroma)  + 31) & (~31);
+        int h = ((init->height>>is_chroma)+6+ 31) & (~31);
 
         pv->ref_stride[i] = w;
 
@@ -2012,8 +2012,8 @@ static int hb_decomb_init( hb_filter_object_t * filter,
     for( i = 0; i < 3; i++ )
     {
         int is_chroma = !!i;
-        int w = ((init->width   + 31) & (~31))>>is_chroma;
-        int h = ((init->height+6+ 31) & (~31))>>is_chroma;
+        int w = ((init->width >>is_chroma)  + 31) & (~31);
+        int h = ((init->height>>is_chroma)+6+ 31) & (~31);
 
         pv->mask[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w;
         pv->mask_filtered[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w;
@@ -2027,8 +2027,8 @@ static int hb_decomb_init( hb_filter_object_t * filter,
         for( i = 0; i < 3; i++ )
         {
             int is_chroma = !!i;
-            int w = ((init->width   + 31) & (~31))>>is_chroma;
-            int h = ((height+6+ 31) & (~31))>>is_chroma;
+            int w = ((init->width>>is_chroma)   + 31) & (~31);
+            int h = ((height>>is_chroma)    + 6 + 31) & (~31);
 
             for( j = 0; j < 4; j++ )
             {
@@ -2037,12 +2037,11 @@ static int hb_decomb_init( hb_filter_object_t * filter,
         }
 
         /* Allocate full-height eedi2 buffers */
-        height = init->height;
         for( i = 0; i < 3; i++ )
         {
             int is_chroma = !!i;
-            int w = ((init->width   + 31) & (~31))>>is_chroma;
-            int h = ((height+6+ 31) & (~31))>>is_chroma;
+            int w = ((init->width >>is_chroma)  + 31) & (~31);
+            int h = ((init->height>>is_chroma)+6+ 31) & (~31);
 
             for( j = 0; j < 5; j++ )
             {
diff --git a/libhb/hb.c b/libhb/hb.c
index 0ad101aaf..9e331f0b3 100644
--- a/libhb/hb.c
+++ b/libhb/hb.c
@@ -865,15 +865,13 @@ void hb_get_preview( hb_handle_t * h, hb_title_t * title, int picture,
  * @param prog_diff   Sensitivity for detecting different colors on progressive frames
  * @param prog_threshold Sensitivity for flagging progressive frames as combed
  */
-int hb_detect_comb( hb_buffer_t * buf, int width, int height, int color_equal, int color_diff, int threshold, int prog_equal, int prog_diff, int prog_threshold )
+int hb_detect_comb( hb_buffer_t * buf, int color_equal, int color_diff, int threshold, int prog_equal, int prog_diff, int prog_threshold )
 {
     int j, k, n, off, cc_1, cc_2, cc[3];
 	// int flag[3] ; // debugging flag
     uint16_t s1, s2, s3, s4;
     cc_1 = 0; cc_2 = 0;
 
-    int offset = 0;
-
     if ( buf->s.flags & 16 )
     {
         /* Frame is progressive, be more discerning. */
@@ -885,21 +883,10 @@ int hb_detect_comb( hb_buffer_t * buf, int width, int height, int color_equal, i
     /* One pas for Y, one pass for Cb, one pass for Cr */    
     for( k = 0; k < 3; k++ )
     {
-        if( k == 1 )
-        {
-            /* Y has already been checked, now offset by Y's dimensions
-               and divide all the other values by 2, since Cr and Cb
-               are half-size compared to Y.                               */
-            offset = width * height;
-            width >>= 1;
-            height >>= 1;
-        }
-        else if ( k == 2 )
-        {
-            /* Y and Cb are done, so the offset needs to be bumped
-               so it's width*height + (width / 2) * (height / 2)  */
-            offset *= 5/4;
-        }
+        uint8_t * data = buf->plane[k].data;
+        int width = buf->plane[k].width;
+        int stride = buf->plane[k].stride;
+        int height = buf->plane[k].height;
 
         for( j = 0; j < width; ++j )
         {
@@ -908,10 +895,10 @@ int hb_detect_comb( hb_buffer_t * buf, int width, int height, int color_equal, i
             for( n = 0; n < ( height - 4 ); n = n + 2 )
             {
                 /* Look at groups of 4 sequential horizontal lines */
-                s1 = ( ( buf->data + offset )[ off + j             ] & 0xff );
-                s2 = ( ( buf->data + offset )[ off + j + width     ] & 0xff );
-                s3 = ( ( buf->data + offset )[ off + j + 2 * width ] & 0xff );
-                s4 = ( ( buf->data + offset )[ off + j + 3 * width ] & 0xff );
+                s1 = ( ( data )[ off + j              ] & 0xff );
+                s2 = ( ( data )[ off + j +     stride ] & 0xff );
+                s3 = ( ( data )[ off + j + 2 * stride ] & 0xff );
+                s4 = ( ( data )[ off + j + 3 * stride ] & 0xff );
 
                 /* Note if the 1st and 2nd lines are more different in
                    color than the 1st and 3rd lines are similar in color.*/
@@ -926,7 +913,7 @@ int hb_detect_comb( hb_buffer_t * buf, int width, int height, int color_equal, i
                         ++cc_2;
 
                 /* Now move down 2 horizontal lines before starting over.*/
-                off += 2 * width;
+                off += 2 * stride;
             }
         }
 
diff --git a/libhb/hb.h b/libhb/hb.h
index a594d93fc..da2311834 100644
--- a/libhb/hb.h
+++ b/libhb/hb.h
@@ -57,7 +57,7 @@ hb_list_t   * hb_get_titles( hb_handle_t * );
 /* hb_detect_comb()
    Analyze a frame for interlacing artifacts, returns true if they're found.
    Taken from Thomas Oestreich's 32detect filter in the Transcode project.  */
-int hb_detect_comb( hb_buffer_t * buf, int width, int height, int color_equal, int color_diff, int threshold, int prog_equal, int prog_diff, int prog_threshold );
+int hb_detect_comb( hb_buffer_t * buf, int color_equal, int color_diff, int threshold, int prog_equal, int prog_diff, int prog_threshold );
 
 int           hb_save_preview( hb_handle_t * h, int title, int preview, 
                                hb_buffer_t *buf );
diff --git a/libhb/scan.c b/libhb/scan.c
index e06f14c5f..4776e08c4 100644
--- a/libhb/scan.c
+++ b/libhb/scan.c
@@ -739,7 +739,7 @@ static int DecodePreviews( hb_scan_t * data, hb_title_t * title )
         }
 
         /* Check preview for interlacing artifacts */
-        if( hb_detect_comb( vid_buf, vid_info.width, vid_info.height, 10, 30, 9, 10, 30, 9 ) )
+        if( hb_detect_comb( vid_buf, 10, 30, 9, 10, 30, 9 ) )
         {
             hb_deep_log( 2, "Interlacing detected in preview frame %i", i+1);
             interlaced_preview_count++;