summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorjstebbins <[email protected]>2012-07-06 23:13:03 +0000
committerjstebbins <[email protected]>2012-07-06 23:13:03 +0000
commita391726398ad581013c2a933bf9e46843dc793f0 (patch)
tree5c44940a236c50b124e907e53b139ef5b3f438a9
parentfd05299c6eb9cd5eb4bf8147b02d51e9242a5276 (diff)
bugfix branch: fix decomb crash
decomb was allocating reference buffers that were too small. This bug appears to have always existed but doesn't usually get triggered because malloc usually rounds allocation sizes up. git-svn-id: svn://svn.handbrake.fr/HandBrake/branches/0.9.x@4813 b64f7644-9d1e-0410-96f1-a4d463321fa5
-rw-r--r--libhb/decomb.c19
1 files changed, 9 insertions, 10 deletions
diff --git a/libhb/decomb.c b/libhb/decomb.c
index b9baa963a..5469313a4 100644
--- a/libhb/decomb.c
+++ b/libhb/decomb.c
@@ -2084,8 +2084,8 @@ hb_filter_private_t * hb_decomb_init( int pix_fmt,
for( i = 0; i < 3; i++ )
{
int is_chroma = !!i;
- int w = ((width + 31) & (~31))>>is_chroma;
- int h = ((height+6+ 31) & (~31))>>is_chroma;
+ int w = ((width >>is_chroma) + 31) & (~31);
+ int h = ((height>>is_chroma)+6+ 31) & (~31);
pv->ref_stride[i] = w;
@@ -2099,8 +2099,8 @@ hb_filter_private_t * hb_decomb_init( int pix_fmt,
for( i = 0; i < 3; i++ )
{
int is_chroma = !!i;
- int w = ((pv->width[0] + 31) & (~31))>>is_chroma;
- int h = ((pv->height[0]+6+ 31) & (~31))>>is_chroma;
+ int w = ((width >>is_chroma) + 31) & (~31);
+ int h = ((height>>is_chroma)+6+ 31) & (~31);
pv->mask[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w;
pv->mask_filtered[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w;
@@ -2110,12 +2110,12 @@ hb_filter_private_t * hb_decomb_init( int pix_fmt,
if( pv->mode & MODE_EEDI2 )
{
/* Allocate half-height eedi2 buffers */
- height = pv->height[0] / 2;
+ int half_h = height / 2;
for( i = 0; i < 3; i++ )
{
int is_chroma = !!i;
- int w = ((width + 31) & (~31))>>is_chroma;
- int h = ((height+6+ 31) & (~31))>>is_chroma;
+ int w = ((width>>is_chroma) + 31) & (~31);
+ int h = ((half_h>>is_chroma)+ 6 + 31) & (~31);
for( j = 0; j < 4; j++ )
{
@@ -2124,12 +2124,11 @@ hb_filter_private_t * hb_decomb_init( int pix_fmt,
}
/* Allocate full-height eedi2 buffers */
- height = pv->height[0];
for( i = 0; i < 3; i++ )
{
int is_chroma = !!i;
- int w = ((width + 31) & (~31))>>is_chroma;
- int h = ((height+6+ 31) & (~31))>>is_chroma;
+ int w = ((width >>is_chroma) + 31) & (~31);
+ int h = ((height>>is_chroma)+6+ 31) & (~31);
for( j = 0; j < 5; j++ )
{