diff options
author | jstebbins <jstebbins.hb@gmail.com> | 2012-07-06 23:12:56 +0000 |
---|---|---|
committer | jstebbins <jstebbins.hb@gmail.com> | 2012-07-06 23:12:56 +0000 |
commit | 5e4a58cc2d39db9c5e8b4f5201c6bf9352412688 (patch) | |
tree | e2ba9652efa8ef449c8d9c91c58f1cb5fc83836b /libhb/decomb.c | |
parent | 4ed21c1ccb42e377ec5a20cf66638b4f207515be (diff) |
libhb: fix comb detection crash and decomb crash
hb_detect_comb() could crash because the dimensions of the video buffer
don't have to match the dimensions returned by work object info() method
if the video has segments of differeing resolutions.
decomb was allocating reference buffers that were too small. This bug appears
to have always existed but doesn't usually get triggered because malloc
usually rounds allocation sizes up.
git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@4812 b64f7644-9d1e-0410-96f1-a4d463321fa5
Diffstat (limited to 'libhb/decomb.c')
-rw-r--r-- | libhb/decomb.c | 17 |
1 files changed, 8 insertions, 9 deletions
diff --git a/libhb/decomb.c b/libhb/decomb.c index 40bcb2a6c..e9519700f 100644 --- a/libhb/decomb.c +++ b/libhb/decomb.c @@ -1997,8 +1997,8 @@ static int hb_decomb_init( hb_filter_object_t * filter, for( i = 0; i < 3; i++ ) { int is_chroma = !!i; - int w = ((init->width + 31) & (~31))>>is_chroma; - int h = ((init->height+6+ 31) & (~31))>>is_chroma; + int w = ((init->width >>is_chroma) + 31) & (~31); + int h = ((init->height>>is_chroma)+6+ 31) & (~31); pv->ref_stride[i] = w; @@ -2012,8 +2012,8 @@ static int hb_decomb_init( hb_filter_object_t * filter, for( i = 0; i < 3; i++ ) { int is_chroma = !!i; - int w = ((init->width + 31) & (~31))>>is_chroma; - int h = ((init->height+6+ 31) & (~31))>>is_chroma; + int w = ((init->width >>is_chroma) + 31) & (~31); + int h = ((init->height>>is_chroma)+6+ 31) & (~31); pv->mask[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w; pv->mask_filtered[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w; @@ -2027,8 +2027,8 @@ static int hb_decomb_init( hb_filter_object_t * filter, for( i = 0; i < 3; i++ ) { int is_chroma = !!i; - int w = ((init->width + 31) & (~31))>>is_chroma; - int h = ((height+6+ 31) & (~31))>>is_chroma; + int w = ((init->width>>is_chroma) + 31) & (~31); + int h = ((height>>is_chroma) + 6 + 31) & (~31); for( j = 0; j < 4; j++ ) { @@ -2037,12 +2037,11 @@ static int hb_decomb_init( hb_filter_object_t * filter, } /* Allocate full-height eedi2 buffers */ - height = init->height; for( i = 0; i < 3; i++ ) { int is_chroma = !!i; - int w = ((init->width + 31) & (~31))>>is_chroma; - int h = ((height+6+ 31) & (~31))>>is_chroma; + int w = ((init->width >>is_chroma) + 31) & (~31); + int h = ((init->height>>is_chroma)+6+ 31) & (~31); for( j = 0; j < 5; j++ ) { |