libhb: fix application of max width/height

In non-anamorphic and custom-anamorphic, if keep-aspect is not set,
don't adjust dimensions to fix aspect when applying max width or height.
If the user is telling us to distort the image, do it.

git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@7362 b64f7644-9d1e-0410-96f1-a4d463321fa5

1 files changed, 12 insertions, 4 deletions

diff --git a/libhb/hb.c b/libhb/hb.c
index a3e373b0f..ab61ff74e 100644
--- a/libhb/hb.c
+++ b/libhb/hb.c
@@ -1003,12 +1003,18 @@ void hb_set_anamorphic_size2(hb_geometry_t *src_geo,
             if (maxWidth && (width > maxWidth))
             {
                 width  = maxWidth;
-                height = MULTIPLE_MOD(width / dar, mod);
+                if (keep_display_aspect)
+                {
+                    height = MULTIPLE_MOD(width / dar, mod);
+                }
             }
             if (maxHeight && (height > maxHeight))
             {
                 height  = maxHeight;
-                width = MULTIPLE_MOD(height * dar, mod);
+                if (keep_display_aspect)
+                {
+                    width = MULTIPLE_MOD(height * dar, mod);
+                }
             }
             dst_par_num = dst_par_den = 1;
         } break;
@@ -1099,7 +1105,8 @@ void hb_set_anamorphic_size2(hb_geometry_t *src_geo,
                 // But otherwise, PAR and DAR will change the least
                 // if we stay as close as possible to the requested
                 // storage aspect.
-                if (!keep_display_aspect)
+                if (!keep_display_aspect &&
+                    (maxHeight == 0 || height < maxHeight))
                 {
                     height = width / storage_aspect + 0.5;
                     height = MULTIPLE_MOD(height, mod);
@@ -1109,7 +1116,8 @@ void hb_set_anamorphic_size2(hb_geometry_t *src_geo,
             {
                 height = maxHeight;
                 // Ditto, see comment above
-                if (!keep_display_aspect)
+                if (!keep_display_aspect &&
+                    (maxWidth == 0 || width < maxWidth))
                 {
                     width = height * storage_aspect + 0.5;
                     width  = MULTIPLE_MOD(width, mod);