Fix int overflow that can happen when computing PAR

If the source has large non-reduced PAR values, our computed value
was overflowing an int.  Compute it in an int64_t then reduce it.
Also, keep num and den below 65535.  Larger values just aren't really
significant and will cause more overflow issues.


git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@3931 b64f7644-9d1e-0410-96f1-a4d463321fa5

3 files changed, 64 insertions, 2 deletions

diff --git a/libhb/common.c b/libhb/common.c
index 4c382e57c..a3887b680 100644
--- a/libhb/common.c
+++ b/libhb/common.c
@@ -412,6 +412,62 @@ void hb_reduce( int *x, int *y, int num, int den )
 }
 
 /**********************************************************************
+ * hb_reduce64
+ **********************************************************************
+ * Given a numerator (num) and a denominator (den), reduce them to an
+ * equivalent fraction and store the result in x and y.
+ *********************************************************************/
+void hb_reduce64( int64_t *x, int64_t *y, int64_t num, int64_t den )
+{
+    // find the greatest common divisor of num & den by Euclid's algorithm
+    int64_t n = num, d = den;
+    while ( d )
+    {
+        int64_t t = d;
+        d = n % d;
+        n = t;
+    }
+
+    // at this point n is the gcd. if it's non-zero remove it from num
+    // and den. Otherwise just return the original values.
+    if ( n )
+    {
+        num /= n;
+        den /= n;
+    }
+
+    *x = num;
+    *y = den;
+
+}
+
+void hb_limit_rational64( int64_t *x, int64_t *y, int64_t num, int64_t den, int64_t limit )
+{
+    hb_reduce64( &num, &den, num, den );
+    if ( num < limit && den < limit )
+    {
+        *x = num;
+        *y = den;
+        return;
+    }
+
+    if ( num > den )
+    {
+        double div = (double)limit / num;
+        num = limit;
+        den *= div;
+    }
+    else
+    {
+        double div = (double)limit / den;
+        den = limit;
+        num *= div;
+    }
+    *x = num;
+    *y = den;
+}
+
+/**********************************************************************
  * hb_fix_aspect
  **********************************************************************
  * Given the output width (if HB_KEEP_WIDTH) or height
diff --git a/libhb/common.h b/libhb/common.h
index bb0c219cb..b945200e7 100644
--- a/libhb/common.h
+++ b/libhb/common.h
@@ -91,6 +91,8 @@ void      * hb_list_item( hb_list_t *, int );
 void        hb_list_close( hb_list_t ** );
 
 void hb_reduce( int *x, int *y, int num, int den );
+void hb_reduce64( int64_t *x, int64_t *y, int64_t num, int64_t den );
+void hb_limit_rational64( int64_t *x, int64_t *y, int64_t num, int64_t den, int64_t limit );
 
 #define HB_KEEP_WIDTH  0
 #define HB_KEEP_HEIGHT 1
diff --git a/libhb/hb.c b/libhb/hb.c
index 55d5f6886..8588f7ff4 100644
--- a/libhb/hb.c
+++ b/libhb/hb.c
@@ -1025,8 +1025,12 @@ void hb_set_anamorphic_size( hb_job_t * job,
             /* The film AR is the source's display width / cropped source height.
                The output display width is the output height * film AR.
                The output PAR is the output display width / output storage width. */
-            pixel_aspect_width = height * cropped_width * pixel_aspect_width;
-            pixel_aspect_height = width * cropped_height * pixel_aspect_height;
+            int64_t par_w, par_h;
+            par_w = (int64_t)height * cropped_width * pixel_aspect_width;
+            par_h = (int64_t)width * cropped_height * pixel_aspect_height;
+            hb_limit_rational64( &par_w, &par_h, par_w, par_h, 65535);
+            pixel_aspect_width = par_w;
+            pixel_aspect_height = par_h;
 
             /* Pass the results back to the caller */
             *output_width = width;