Merge pull request #19 from jstebbins/ana-custom

Fix bizarre custom anamorphic behavior

4 files changed, 155 insertions, 78 deletions

diff --git a/gtk/src/callbacks.c b/gtk/src/callbacks.c
index 215900275..e77e90de4 100644
--- a/gtk/src/callbacks.c
+++ b/gtk/src/callbacks.c
@@ -1873,7 +1873,7 @@ set_title_settings(signal_user_data_t *ud, GhbValue *settings)
         if (!(keep_aspect || pic_par) || pic_par == 3)
         {
             ghb_dict_set_int(settings, "scale_height",
-                             title->geometry.width - title->crop[0] - title->crop[1]);
+                             title->geometry.height - title->crop[0] - title->crop[1]);
         }
 
         ghb_set_scale_settings(settings, GHB_PIC_KEEP_PAR|GHB_PIC_USE_MAX);
@@ -2501,7 +2501,7 @@ scale_width_changed_cb(GtkWidget *widget, signal_user_data_t *ud)
     ghb_check_dependency(ud, widget, NULL);
     ghb_clear_presets_selection(ud);
     if (gtk_widget_is_sensitive(widget))
-        ghb_set_scale(ud, GHB_PIC_KEEP_WIDTH);
+        ghb_set_scale(ud, GHB_PIC_KEEP_WIDTH|GHB_PIC_KEEP_PAR);
     update_preview = TRUE;
     ghb_live_reset(ud);
 
@@ -2516,7 +2516,7 @@ scale_height_changed_cb(GtkWidget *widget, signal_user_data_t *ud)
     ghb_check_dependency(ud, widget, NULL);
     ghb_clear_presets_selection(ud);
     if (gtk_widget_is_sensitive(widget))
-        ghb_set_scale(ud, GHB_PIC_KEEP_HEIGHT);
+        ghb_set_scale(ud, GHB_PIC_KEEP_HEIGHT|GHB_PIC_KEEP_PAR);
 
     update_preview = TRUE;
     ghb_live_reset(ud);
@@ -2532,7 +2532,7 @@ crop_changed_cb(GtkWidget *widget, signal_user_data_t *ud)
     ghb_check_dependency(ud, widget, NULL);
     ghb_clear_presets_selection(ud);
     if (gtk_widget_is_sensitive(widget))
-        ghb_set_scale(ud, 0);
+        ghb_set_scale(ud, GHB_PIC_KEEP_PAR);
     update_preview = TRUE;
     ghb_live_reset(ud);
 
@@ -2562,7 +2562,7 @@ display_height_changed_cb(GtkWidget *widget, signal_user_data_t *ud)
     ghb_clear_presets_selection(ud);
     ghb_live_reset(ud);
     if (gtk_widget_is_sensitive(widget))
-        ghb_set_scale(ud, GHB_PIC_KEEP_DISPLAY_HEIGHT);
+        ghb_set_scale(ud, GHB_PIC_KEEP_DISPLAY_HEIGHT|GHB_PIC_KEEP_PAR);
 
     update_preview = TRUE;
 }
@@ -2590,7 +2590,7 @@ scale_changed_cb(GtkWidget *widget, signal_user_data_t *ud)
     ghb_clear_presets_selection(ud);
     ghb_live_reset(ud);
     if (gtk_widget_is_sensitive(widget))
-        ghb_set_scale(ud, 0);
+        ghb_set_scale(ud, GHB_PIC_KEEP_PAR);
     update_preview = TRUE;
 
     update_aspect_info(ud);
@@ -2604,7 +2604,7 @@ show_crop_changed_cb(GtkWidget *widget, signal_user_data_t *ud)
     ghb_check_dependency(ud, widget, NULL);
     ghb_live_reset(ud);
     if (gtk_widget_is_sensitive(widget))
-        ghb_set_scale(ud, 0);
+        ghb_set_scale(ud, GHB_PIC_KEEP_PAR);
     ghb_pref_save(ud->prefs, "preview_show_crop");
     update_preview = TRUE;
 }
diff --git a/gtk/src/hb-backend.c b/gtk/src/hb-backend.c
index beb9797e5..88e7a89a1 100644
--- a/gtk/src/hb-backend.c
+++ b/gtk/src/hb-backend.c
@@ -3746,6 +3746,8 @@ ghb_set_scale_settings(GhbValue *settings, gint mode)
                 uiGeo.geometry.par.num =
                         ghb_dict_get_int(settings, "PictureDisplayWidth");
                 uiGeo.geometry.par.den = width;
+                hb_reduce(&uiGeo.geometry.par.num, &uiGeo.geometry.par.den,
+                           uiGeo.geometry.par.num,  uiGeo.geometry.par.den);
             }
         }
         else
@@ -3881,9 +3883,10 @@ get_preview_geometry(signal_user_data_t *ud, const hb_title_t *title,
     srcGeo->par    = title->geometry.par;
 
     uiGeo->mode = ghb_settings_combo_int(ud->settings, "PicturePAR");
-    uiGeo->keep = ghb_dict_get_bool(ud->settings, "PictureKeepRatio") ||
-                                        uiGeo->mode == HB_ANAMORPHIC_STRICT  ||
-                                        uiGeo->mode == HB_ANAMORPHIC_LOOSE;
+    uiGeo->keep = (ghb_dict_get_bool(ud->settings, "PictureKeepRatio") ||
+                                 uiGeo->mode == HB_ANAMORPHIC_STRICT  ||
+                                 uiGeo->mode == HB_ANAMORPHIC_LOOSE) ?
+                                 HB_KEEP_DISPLAY_ASPECT : 0;
     uiGeo->itu_par = 0;
     uiGeo->modulus = ghb_settings_combo_int(ud->settings, "PictureModulus");
     uiGeo->crop[0] = ghb_dict_get_int(ud->settings, "PictureTopCrop");
@@ -4557,9 +4560,11 @@ ghb_get_preview_image(
         src_line += image->plane[0].stride;
         dst += stride;
     }
-    gint w = ghb_dict_get_int(ud->settings, "scale_width");
-    gint h = ghb_dict_get_int(ud->settings, "scale_height");
-    ghb_par_scale(ud, &w, &h, resultGeo.par.num, resultGeo.par.den);
+
+    *out_width = ghb_dict_get_int(ud->settings, "scale_width");
+    *out_height = ghb_dict_get_int(ud->settings, "scale_height");
+    ghb_par_scale(ud, out_width, out_height,
+                  resultGeo.par.num, resultGeo.par.den);
 
     gint c0, c1, c2, c3;
     c0 = ghb_dict_get_int(ud->settings, "PictureTopCrop");
@@ -4567,11 +4572,17 @@ ghb_get_preview_image(
     c2 = ghb_dict_get_int(ud->settings, "PictureLeftCrop");
     c3 = ghb_dict_get_int(ud->settings, "PictureRightCrop");
 
-    gdouble xscale = (gdouble)w / (gdouble)(title->geometry.width - c2 - c3);
-    gdouble yscale = (gdouble)h / (gdouble)(title->geometry.height - c0 - c1);
-
-    *out_width = w;
-    *out_height = h;
+    gdouble xscale, yscale;
+    if (ghb_dict_get_bool(ud->prefs, "preview_show_crop"))
+    {
+        xscale = (gdouble)image->width / title->geometry.width;
+        yscale = (gdouble)image->height / title->geometry.height;
+    }
+    else
+    {
+        xscale = (gdouble)image->width / (title->geometry.width - c2 - c3);
+        yscale = (gdouble)image->height / (title->geometry.height - c0 - c1);
+    }
 
     int previewWidth = image->width;
     int previewHeight = image->height;
@@ -4579,14 +4590,16 @@ ghb_get_preview_image(
     // If the preview is too large to fit the screen, reduce it's size.
     if (ghb_dict_get_bool(ud->prefs, "reduce_hd_preview"))
     {
-        GdkScreen *ss;
-        gint s_w, s_h;
         gint factor = 80;
 
         if (ghb_dict_get_bool(ud->prefs, "preview_fullscreen"))
         {
             factor = 100;
         }
+
+        GdkScreen *ss;
+        gint s_w, s_h;
+
         ss = gdk_screen_get_default();
         s_w = gdk_screen_get_width(ss);
         s_h = gdk_screen_get_height(ss);
@@ -4610,28 +4623,28 @@ ghb_get_preview_image(
             }
             xscale *= (gdouble)previewWidth / orig_w;
             yscale *= (gdouble)previewHeight / orig_h;
-            w *= (gdouble)previewWidth / orig_w;
-            h *= (gdouble)previewHeight / orig_h;
             scaled_preview = gdk_pixbuf_scale_simple(preview,
                             previewWidth, previewHeight, GDK_INTERP_HYPER);
             g_object_unref(preview);
             preview = scaled_preview;
         }
     }
+
     if (ghb_dict_get_bool(ud->prefs, "preview_show_crop"))
     {
         c0 *= yscale;
         c1 *= yscale;
         c2 *= xscale;
         c3 *= xscale;
+
         // Top
-        hash_pixbuf(preview, c2, 0, w, c0, 32, 0);
+        hash_pixbuf(preview, 0, 0, previewWidth, c0, 32, 0);
         // Bottom
-        hash_pixbuf(preview, c2, previewHeight-c1, w, c1, 32, 0);
+        hash_pixbuf(preview, 0, previewHeight-c1, previewWidth, c1, 32, 0);
         // Left
-        hash_pixbuf(preview, 0, c0, c2, h, 32, 1);
+        hash_pixbuf(preview, 0, 0, c2, previewHeight, 32, 1);
         // Right
-        hash_pixbuf(preview, previewWidth-c3, c0, c3, h, 32, 1);
+        hash_pixbuf(preview, previewWidth-c3, 0, c3, previewHeight, 32, 1);
     }
     hb_image_close(&image);
     return preview;
diff --git a/libhb/common.h b/libhb/common.h
index 6811e1eca..a78f86475 100644
--- a/libhb/common.h
+++ b/libhb/common.h
@@ -72,6 +72,11 @@
 
 #define HB_DVD_READ_BUFFER_SIZE 2048
 
+#define HB_MIN_WIDTH    32
+#define HB_MIN_HEIGHT   32
+#define HB_MAX_WIDTH    20480
+#define HB_MAX_HEIGHT   20480
+
 typedef struct hb_handle_s hb_handle_t;
 typedef struct hb_hwd_s hb_hwd_t;
 typedef struct hb_list_s hb_list_t;
diff --git a/libhb/hb.c b/libhb/hb.c
index 5f688c012..2a90726ce 100644
--- a/libhb/hb.c
+++ b/libhb/hb.c
@@ -758,7 +758,8 @@ hb_image_t* hb_get_preview2(hb_handle_t * h, int title_idx, int picture,
                             hb_geometry_settings_t *geo, int deinterlace)
 {
     char                 filename[1024];
-    hb_buffer_t        * in_buf, * deint_buf = NULL, * preview_buf;
+    hb_buffer_t        * in_buf = NULL, * deint_buf = NULL;
+    hb_buffer_t        * preview_buf = NULL;
     uint32_t             swsflags;
     AVPicture            pic_in, pic_preview, pic_deint, pic_crop;
     struct SwsContext  * context;
@@ -767,6 +768,15 @@ hb_image_t* hb_get_preview2(hb_handle_t * h, int title_idx, int picture,
                 geo->geometry.par.num / geo->geometry.par.den;
     int height = geo->geometry.height;
 
+    // Set min/max dimensions to prevent failure to initialize
+    // sws context and absurd sizes.
+    //
+    // This means output image size may not match requested image size!
+    int ww = width, hh = height;
+    width  = MIN(MAX(width,                HB_MIN_WIDTH),  HB_MAX_WIDTH);
+    height = MIN(MAX(height * width  / ww, HB_MIN_HEIGHT), HB_MAX_HEIGHT);
+    width  = MIN(MAX(width  * height / hh, HB_MIN_WIDTH),  HB_MAX_WIDTH);
+
     swsflags = SWS_LANCZOS | SWS_ACCURATE_RND;
 
     preview_buf = hb_frame_buffer_init(AV_PIX_FMT_RGB32, width, height);
@@ -816,6 +826,12 @@ hb_image_t* hb_get_preview2(hb_handle_t * h, int title_idx, int picture,
                 title->geometry.height - (geo->crop[0] + geo->crop[1]),
                 AV_PIX_FMT_YUV420P, width, height, AV_PIX_FMT_RGB32, swsflags);
 
+    if (context == NULL)
+    {
+        // if by chance hb_sws_get_context fails, don't crash in sws_scale
+        goto fail;
+    }
+
     // Scale
     sws_scale(context,
               (const uint8_t* const *)pic_crop.data, pic_crop.linesize,
@@ -836,6 +852,10 @@ hb_image_t* hb_get_preview2(hb_handle_t * h, int title_idx, int picture,
 
 fail:
 
+    hb_buffer_close( &in_buf );
+    hb_buffer_close( &deint_buf );
+    hb_buffer_close( &preview_buf );
+
     image = hb_image_init(AV_PIX_FMT_RGB32, width, height);
     return image;
 }
@@ -1031,12 +1051,24 @@ void hb_set_anamorphic_size2(hb_geometry_t *src_geo,
     int width, height;
     int maxWidth, maxHeight;
 
-    maxWidth = MULTIPLE_MOD_DOWN(geo->maxWidth, mod);
-    maxHeight = MULTIPLE_MOD_DOWN(geo->maxHeight, mod);
-    if (maxWidth && maxWidth < 32)
-        maxWidth = 32;
-    if (maxHeight && maxHeight < 32)
-        maxHeight = 32;
+    if (geo->maxWidth > 0)
+    {
+        maxWidth  = MIN(MAX(MULTIPLE_MOD_DOWN(geo->maxWidth, mod),
+                            HB_MIN_WIDTH), HB_MAX_WIDTH);
+    }
+    else
+    {
+        maxWidth  = HB_MAX_WIDTH;
+    }
+    if (geo->maxHeight > 0)
+    {
+        maxHeight = MIN(MAX(MULTIPLE_MOD_DOWN(geo->maxHeight, mod),
+                            HB_MIN_HEIGHT), HB_MAX_HEIGHT);
+    }
+    else
+    {
+        maxHeight = HB_MAX_HEIGHT;
+    }
 
     switch (geo->mode)
     {
@@ -1069,7 +1101,25 @@ void hb_set_anamorphic_size2(hb_geometry_t *src_geo,
                 width = MULTIPLE_MOD_UP(geo->geometry.width, mod);
                 height = MULTIPLE_MOD_UP(geo->geometry.height, mod);
             }
-            if (maxWidth && (width > maxWidth))
+
+            // Limit to min/max dimensions
+            if (width < HB_MIN_WIDTH)
+            {
+                width  = HB_MIN_WIDTH;
+                if (keep_display_aspect)
+                {
+                    height = MULTIPLE_MOD(width / dar, mod);
+                }
+            }
+            if (height < HB_MIN_HEIGHT)
+            {
+                height  = HB_MIN_HEIGHT;
+                if (keep_display_aspect)
+                {
+                    width = MULTIPLE_MOD(height * dar, mod);
+                }
+            }
+            if (width > maxWidth)
             {
                 width  = maxWidth;
                 if (keep_display_aspect)
@@ -1077,7 +1127,7 @@ void hb_set_anamorphic_size2(hb_geometry_t *src_geo,
                     height = MULTIPLE_MOD(width / dar, mod);
                 }
             }
-            if (maxHeight && (height > maxHeight))
+            if (height > maxHeight)
             {
                 height  = maxHeight;
                 if (keep_display_aspect)
@@ -1132,13 +1182,23 @@ void hb_set_anamorphic_size2(hb_geometry_t *src_geo,
                 width = MULTIPLE_MOD_UP(height * storage_aspect + 0.5, mod);
             }
 
-            if (maxWidth && (maxWidth < width))
+            // Limit to min/max dimensions
+            if (width < HB_MIN_WIDTH)
+            {
+                width  = HB_MIN_WIDTH;
+                height = MULTIPLE_MOD(width / storage_aspect + 0.5, mod);
+            }
+            if (height < HB_MIN_HEIGHT)
+            {
+                height  = HB_MIN_HEIGHT;
+                width = MULTIPLE_MOD(height * storage_aspect + 0.5, mod);
+            }
+            if (width > maxWidth)
             {
                 width = maxWidth;
                 height = MULTIPLE_MOD(width / storage_aspect + 0.5, mod);
             }
-
-            if (maxHeight && (maxHeight < height))
+            if (height > maxHeight)
             {
                 height = maxHeight;
                 width = MULTIPLE_MOD(height * storage_aspect + 0.5, mod);
@@ -1155,42 +1215,26 @@ void hb_set_anamorphic_size2(hb_geometry_t *src_geo,
             /* Anamorphic 3: Power User Jamboree
                - Set everything based on specified values */
 
-            /* Use specified storage dimensions */
-            storage_aspect = (double)geo->geometry.width / geo->geometry.height;
-
             /* Time to get picture dimensions that divide cleanly.*/
             width  = MULTIPLE_MOD_UP(geo->geometry.width, mod);
             height = MULTIPLE_MOD_UP(geo->geometry.height, mod);
 
-            /* Bind to max dimensions */
-            if (maxWidth && width > maxWidth)
+            // Limit to min/max dimensions
+            if (width < HB_MIN_WIDTH)
+            {
+                width  = HB_MIN_WIDTH;
+            }
+            if (height < HB_MIN_HEIGHT)
+            {
+                height  = HB_MIN_HEIGHT;
+            }
+            if (width > maxWidth)
             {
                 width = maxWidth;
-                // If we are keeping the display aspect, then we are going
-                // to be modifying the PAR anyway.  So it's preferred
-                // to let the width/height stray some from the original
-                // requested storage aspect.
-                //
-                // But otherwise, PAR and DAR will change the least
-                // if we stay as close as possible to the requested
-                // storage aspect.
-                if (!keep_display_aspect &&
-                    (maxHeight == 0 || height < maxHeight))
-                {
-                    height = width / storage_aspect + 0.5;
-                    height = MULTIPLE_MOD(height, mod);
-                }
             }
-            if (maxHeight && height > maxHeight)
+            if (height > maxHeight)
             {
                 height = maxHeight;
-                // Ditto, see comment above
-                if (!keep_display_aspect &&
-                    (maxWidth == 0 || width < maxWidth))
-                {
-                    width = height * storage_aspect + 0.5;
-                    width  = MULTIPLE_MOD(width, mod);
-                }
             }
             if (keep_display_aspect)
             {
@@ -1203,21 +1247,36 @@ void hb_set_anamorphic_size2(hb_geometry_t *src_geo,
                 dst_par_den = (int64_t)width  * cropped_height *
                                        src_par.den;
             }
-            else
-            {
-                /* If the dimensions were changed by the modulus
-                 * or by maxWidth/maxHeight, we also change the
-                 * output PAR so that the DAR is unchanged.
-                 *
-                 * PAR is the requested output display width / storage width
-                 * requested output display width is the original
-                 * requested width * original requested PAR
-                 */
-                dst_par_num = geo->geometry.width * dst_par_num;
-                dst_par_den = width * dst_par_den;
-            }
         } break;
     }
+    if (width < HB_MIN_WIDTH || height < HB_MIN_HEIGHT ||
+        width > maxWidth     || height > maxHeight)
+    {
+        // All limits set above also attempted to keep PAR and DAR.
+        // If we are still outside limits, enforce them and modify
+        // PAR to keep DAR
+        if (width < HB_MIN_WIDTH)
+        {
+            width  = HB_MIN_WIDTH;
+        }
+        if (height < HB_MIN_HEIGHT)
+        {
+            height  = HB_MIN_HEIGHT;
+        }
+        if (width > maxWidth)
+        {
+            width = maxWidth;
+        }
+        if (height > maxHeight)
+        {
+            height = maxHeight;
+        }
+        if (keep_display_aspect && geo->mode != HB_ANAMORPHIC_NONE)
+        {
+            dst_par_num = (int64_t)height * cropped_width  * src_par.num;
+            dst_par_den = (int64_t)width  * cropped_height * src_par.den;
+        }
+    }
 
     /* Pass the results back to the caller */
     result->width = width;