summaryrefslogtreecommitdiffstats
path: root/libhb/decomb.c
diff options
context:
space:
mode:
authorjstebbins <jstebbins.hb@gmail.com>2012-07-06 23:12:56 +0000
committerjstebbins <jstebbins.hb@gmail.com>2012-07-06 23:12:56 +0000
commit5e4a58cc2d39db9c5e8b4f5201c6bf9352412688 (patch)
treee2ba9652efa8ef449c8d9c91c58f1cb5fc83836b /libhb/decomb.c
parent4ed21c1ccb42e377ec5a20cf66638b4f207515be (diff)
libhb: fix comb detection crash and decomb crash
hb_detect_comb() could crash because the dimensions of the video buffer don't have to match the dimensions returned by work object info() method if the video has segments of differeing resolutions. decomb was allocating reference buffers that were too small. This bug appears to have always existed but doesn't usually get triggered because malloc usually rounds allocation sizes up. git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@4812 b64f7644-9d1e-0410-96f1-a4d463321fa5
Diffstat (limited to 'libhb/decomb.c')
-rw-r--r--libhb/decomb.c17
1 files changed, 8 insertions, 9 deletions
diff --git a/libhb/decomb.c b/libhb/decomb.c
index 40bcb2a6c..e9519700f 100644
--- a/libhb/decomb.c
+++ b/libhb/decomb.c
@@ -1997,8 +1997,8 @@ static int hb_decomb_init( hb_filter_object_t * filter,
for( i = 0; i < 3; i++ )
{
int is_chroma = !!i;
- int w = ((init->width + 31) & (~31))>>is_chroma;
- int h = ((init->height+6+ 31) & (~31))>>is_chroma;
+ int w = ((init->width >>is_chroma) + 31) & (~31);
+ int h = ((init->height>>is_chroma)+6+ 31) & (~31);
pv->ref_stride[i] = w;
@@ -2012,8 +2012,8 @@ static int hb_decomb_init( hb_filter_object_t * filter,
for( i = 0; i < 3; i++ )
{
int is_chroma = !!i;
- int w = ((init->width + 31) & (~31))>>is_chroma;
- int h = ((init->height+6+ 31) & (~31))>>is_chroma;
+ int w = ((init->width >>is_chroma) + 31) & (~31);
+ int h = ((init->height>>is_chroma)+6+ 31) & (~31);
pv->mask[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w;
pv->mask_filtered[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w;
@@ -2027,8 +2027,8 @@ static int hb_decomb_init( hb_filter_object_t * filter,
for( i = 0; i < 3; i++ )
{
int is_chroma = !!i;
- int w = ((init->width + 31) & (~31))>>is_chroma;
- int h = ((height+6+ 31) & (~31))>>is_chroma;
+ int w = ((init->width>>is_chroma) + 31) & (~31);
+ int h = ((height>>is_chroma) + 6 + 31) & (~31);
for( j = 0; j < 4; j++ )
{
@@ -2037,12 +2037,11 @@ static int hb_decomb_init( hb_filter_object_t * filter,
}
/* Allocate full-height eedi2 buffers */
- height = init->height;
for( i = 0; i < 3; i++ )
{
int is_chroma = !!i;
- int w = ((init->width + 31) & (~31))>>is_chroma;
- int h = ((height+6+ 31) & (~31))>>is_chroma;
+ int w = ((init->width >>is_chroma) + 31) & (~31);
+ int h = ((init->height>>is_chroma)+6+ 31) & (~31);
for( j = 0; j < 5; j++ )
{