diff options
author | jstebbins <[email protected]> | 2012-07-06 23:13:03 +0000 |
---|---|---|
committer | jstebbins <[email protected]> | 2012-07-06 23:13:03 +0000 |
commit | a391726398ad581013c2a933bf9e46843dc793f0 (patch) | |
tree | 5c44940a236c50b124e907e53b139ef5b3f438a9 | |
parent | fd05299c6eb9cd5eb4bf8147b02d51e9242a5276 (diff) |
bugfix branch: fix decomb crash
decomb was allocating reference buffers that were too small. This bug appears
to have always existed but doesn't usually get triggered because malloc
usually rounds allocation sizes up.
git-svn-id: svn://svn.handbrake.fr/HandBrake/branches/0.9.x@4813 b64f7644-9d1e-0410-96f1-a4d463321fa5
-rw-r--r-- | libhb/decomb.c | 19 |
1 files changed, 9 insertions, 10 deletions
diff --git a/libhb/decomb.c b/libhb/decomb.c index b9baa963a..5469313a4 100644 --- a/libhb/decomb.c +++ b/libhb/decomb.c @@ -2084,8 +2084,8 @@ hb_filter_private_t * hb_decomb_init( int pix_fmt, for( i = 0; i < 3; i++ ) { int is_chroma = !!i; - int w = ((width + 31) & (~31))>>is_chroma; - int h = ((height+6+ 31) & (~31))>>is_chroma; + int w = ((width >>is_chroma) + 31) & (~31); + int h = ((height>>is_chroma)+6+ 31) & (~31); pv->ref_stride[i] = w; @@ -2099,8 +2099,8 @@ hb_filter_private_t * hb_decomb_init( int pix_fmt, for( i = 0; i < 3; i++ ) { int is_chroma = !!i; - int w = ((pv->width[0] + 31) & (~31))>>is_chroma; - int h = ((pv->height[0]+6+ 31) & (~31))>>is_chroma; + int w = ((width >>is_chroma) + 31) & (~31); + int h = ((height>>is_chroma)+6+ 31) & (~31); pv->mask[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w; pv->mask_filtered[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w; @@ -2110,12 +2110,12 @@ hb_filter_private_t * hb_decomb_init( int pix_fmt, if( pv->mode & MODE_EEDI2 ) { /* Allocate half-height eedi2 buffers */ - height = pv->height[0] / 2; + int half_h = height / 2; for( i = 0; i < 3; i++ ) { int is_chroma = !!i; - int w = ((width + 31) & (~31))>>is_chroma; - int h = ((height+6+ 31) & (~31))>>is_chroma; + int w = ((width>>is_chroma) + 31) & (~31); + int h = ((half_h>>is_chroma)+ 6 + 31) & (~31); for( j = 0; j < 4; j++ ) { @@ -2124,12 +2124,11 @@ hb_filter_private_t * hb_decomb_init( int pix_fmt, } /* Allocate full-height eedi2 buffers */ - height = pv->height[0]; for( i = 0; i < 3; i++ ) { int is_chroma = !!i; - int w = ((width + 31) & (~31))>>is_chroma; - int h = ((height+6+ 31) & (~31))>>is_chroma; + int w = ((width >>is_chroma) + 31) & (~31); + int h = ((height>>is_chroma)+6+ 31) & (~31); for( j = 0; j < 5; j++ ) { |