Add ability to apply filters to previews

New APIs hb_get_preview3 and JSON version hb_get_preview3_json to
retrieve a cached preview image and apply relevant filters from an
hb_job_t to the image.

Returned image also has PAR applied, i.e. PAR of image is 1:1

8 files changed, 407 insertions, 100 deletions

diff --git a/libhb/common.c b/libhb/common.c
index 0be4bced5..071bcf6cd 100644
--- a/libhb/common.c
+++ b/libhb/common.c
@@ -5914,3 +5914,100 @@ int hb_get_bit_depth(int format)
 
     return max;
 }
+
+static int bit_depth_is_supported(hb_job_t * job, int bit_depth)
+{
+    for (int i = 0; i < hb_list_count(job->list_filter); i++)
+    {
+        hb_filter_object_t *filter = hb_list_item(job->list_filter, i);
+
+        switch (filter->id) {
+            case HB_FILTER_DETELECINE:
+            case HB_FILTER_COMB_DETECT:
+            case HB_FILTER_DECOMB:
+            case HB_FILTER_DENOISE:
+            case HB_FILTER_NLMEANS:
+            case HB_FILTER_CHROMA_SMOOTH:
+            case HB_FILTER_LAPSHARP:
+            case HB_FILTER_UNSHARP:
+            case HB_FILTER_GRAYSCALE:
+                return 0;
+        }
+    }
+
+    if (hb_video_encoder_get_depth(job->vcodec) < bit_depth)
+    {
+        return 0;
+    }
+
+    return 1;
+}
+
+static int pix_fmt_is_supported(hb_job_t * job, int pix_fmt)
+{
+    for (int i = 0; i < hb_list_count(job->list_filter); i++)
+    {
+        hb_filter_object_t *filter = hb_list_item(job->list_filter, i);
+
+        switch (filter->id)
+        {
+            case HB_FILTER_DETELECINE:
+            case HB_FILTER_COMB_DETECT:
+            case HB_FILTER_DECOMB:
+            case HB_FILTER_DENOISE:
+            case HB_FILTER_NLMEANS:
+            case HB_FILTER_CHROMA_SMOOTH:
+            case HB_FILTER_LAPSHARP:
+            case HB_FILTER_UNSHARP:
+            case HB_FILTER_GRAYSCALE:
+                if (pix_fmt != AV_PIX_FMT_YUV420P)
+                {
+                    return 0;
+                }
+        }
+    }
+
+    return 1;
+}
+
+int hb_get_best_pix_fmt(hb_job_t * job)
+{
+    int bit_depth = hb_get_bit_depth(job->title->pix_fmt);
+#if HB_PROJECT_FEATURE_QSV && (defined( _WIN32 ) || defined( __MINGW32__ ))
+    if (hb_qsv_encoder_info_get(hb_qsv_get_adapter_index(), job->vcodec))
+    {
+        if (hb_qsv_full_path_is_enabled(job))
+        {
+            if (job->title->pix_fmt == AV_PIX_FMT_YUV420P10 && job->vcodec == HB_VCODEC_QSV_H265_10BIT)
+            {
+                return AV_PIX_FMT_P010LE;
+            }
+            else
+            {
+                return AV_PIX_FMT_NV12;
+            }
+        }
+        else
+        {
+            // system memory usage: QSV encoder only or QSV decoder + SW filters + QSV encoder
+            return AV_PIX_FMT_YUV420P;
+        }
+    }
+#endif
+    if (job->vcodec == HB_VCODEC_FFMPEG_MF_H264 || job->vcodec == HB_VCODEC_FFMPEG_MF_H265)
+    {
+        if (pix_fmt_is_supported(job, AV_PIX_FMT_NV12))
+        {
+            return AV_PIX_FMT_NV12;
+        }
+    }
+    if (bit_depth >= 12 && bit_depth_is_supported(job, 12))
+    {
+        return AV_PIX_FMT_YUV420P12;
+    }
+    if (bit_depth >= 10 && bit_depth_is_supported(job, 10))
+    {
+        return AV_PIX_FMT_YUV420P10;
+    }
+    return AV_PIX_FMT_YUV420P;
+}
diff --git a/libhb/cropscale.c b/libhb/cropscale.c
index 1c79cac22..ed6e07fcf 100644
--- a/libhb/cropscale.c
+++ b/libhb/cropscale.c
@@ -63,7 +63,8 @@ static int crop_scale_init(hb_filter_object_t * filter, hb_filter_init_t * init)
 
     hb_dict_t        * settings = filter->settings;
     hb_value_array_t * avfilters = hb_value_array_init();
-    int                width, height, top, bottom, left, right;
+    int                width, height;
+    int                top = 0, bottom = 0, left = 0, right = 0;
     const char       * matrix;
 
     // Convert crop settings to 'crop' avfilter
@@ -84,6 +85,9 @@ static int crop_scale_init(hb_filter_object_t * filter, hb_filter_init_t * init)
         hb_value_array_append(avfilters, avfilter);
     }
 
+    width  = init->geometry.width;
+    height = init->geometry.height;
+
     // Convert scale settings to 'scale' avfilter
     hb_dict_extract_int(&width, settings, "width");
     hb_dict_extract_int(&height, settings, "height");
@@ -183,7 +187,21 @@ static int crop_scale_init(hb_filter_object_t * filter, hb_filter_init_t * init)
     if (!hb_qsv_hw_filters_are_enabled(init->job))
 #endif
     {
-        hb_dict_set(avsettings, "pix_fmts", hb_value_string(av_get_pix_fmt_name(init->pix_fmt)));
+        char * out_pix_fmt = NULL;
+
+        // "out_pix_fmt" is a private option used internally by
+        // handbrake for preview generation
+        hb_dict_extract_string(&out_pix_fmt, settings, "out_pix_fmt");
+        if (out_pix_fmt != NULL)
+        {
+            hb_dict_set_string(avsettings, "pix_fmts", out_pix_fmt);
+            free(out_pix_fmt);
+        }
+        else
+        {
+            hb_dict_set_string(avsettings, "pix_fmts",
+                av_get_pix_fmt_name(init->pix_fmt));
+        }
         hb_dict_set(avfilter, "format", avsettings);
         hb_value_array_append(avfilters, avfilter);
     }
diff --git a/libhb/handbrake/common.h b/libhb/handbrake/common.h
index 21d1113b0..8d4de7246 100644
--- a/libhb/handbrake/common.h
+++ b/libhb/handbrake/common.h
@@ -1483,6 +1483,7 @@ int hb_output_color_transfer(hb_job_t * job);
 int hb_output_color_matrix(hb_job_t * job);
 
 int hb_get_bit_depth(int format);
+int hb_get_best_pix_fmt(hb_job_t * job);
 
 #define HB_NEG_FLOAT_REG "(([-])?(([0-9]+([.,][0-9]+)?)|([.,][0-9]+))"
 #define HB_FLOAT_REG     "(([0-9]+([.,][0-9]+)?)|([.,][0-9]+))"
diff --git a/libhb/handbrake/handbrake.h b/libhb/handbrake/handbrake.h
index dd03b5e9f..af66aa394 100644
--- a/libhb/handbrake/handbrake.h
+++ b/libhb/handbrake/handbrake.h
@@ -77,6 +77,8 @@ hb_buffer_t * hb_read_preview( hb_handle_t * h, hb_title_t *title,
 
 hb_image_t  * hb_get_preview2(hb_handle_t * h, int title_idx, int picture,
                               hb_geometry_settings_t *geo, int deinterlace);
+hb_image_t  * hb_get_preview3(hb_handle_t * h, int picture,
+                              hb_dict_t * job_dict);
 void          hb_set_anamorphic_size2(hb_geometry_t *src_geo,
                                       hb_geometry_settings_t *geo,
                                       hb_geometry_t *result);
diff --git a/libhb/handbrake/hb_json.h b/libhb/handbrake/hb_json.h
index c176095a0..4050c3ffc 100644
--- a/libhb/handbrake/hb_json.h
+++ b/libhb/handbrake/hb_json.h
@@ -36,6 +36,7 @@ hb_image_t * hb_json_to_image(char *json_image);
 char       * hb_get_preview_params_json(int title_idx, int preview_idx,
                             int deinterlace, hb_geometry_settings_t *settings);
 char       * hb_get_preview_json(hb_handle_t * h, const char *json_param);
+hb_image_t * hb_get_preview3_json(hb_handle_t * h, int picture, const char *json_job);
 void         hb_json_job_scan( hb_handle_t * h, const char * json_job );
 hb_dict_t  * hb_version_dict(void);
 
diff --git a/libhb/hb.c b/libhb/hb.c
index ea554207a..b748f8b0e 100644
--- a/libhb/hb.c
+++ b/libhb/hb.c
@@ -9,6 +9,7 @@
 
 #include "handbrake/handbrake.h"
 #include "handbrake/hbffmpeg.h"
+#include "handbrake/hbavfilter.h"
 #include "handbrake/encx264.h"
 #include "libavfilter/avfilter.h"
 #include <stdio.h>
@@ -594,9 +595,13 @@ hb_buffer_t * hb_read_preview(hb_handle_t * h, hb_title_t *title, int preview, i
     hb_buffer_t * buf;
     buf = hb_frame_buffer_init(AV_PIX_FMT_YUV420P,
                                title->geometry.width, title->geometry.height);
+    buf->f.color_prim     = title->color_prim;
+    buf->f.color_transfer = title->color_transfer;
+    buf->f.color_matrix   = title->color_matrix;
 
     if (!buf)
     {
+        hb_error("hb_read_preview: hb_frame_buffer_init failed");
         goto done;
     }
 
@@ -822,6 +827,274 @@ fail:
     return image;
 }
 
+static void process_filter(hb_filter_object_t * filter)
+{
+    hb_buffer_t * in, * out;
+
+    while (filter->status != HB_FILTER_DONE)
+    {
+        in = hb_fifo_get_wait(filter->fifo_in);
+        if (in == NULL)
+            continue;
+
+        out = NULL;
+        filter->status = filter->work(filter, &in, &out);
+        if (in != NULL)
+        {
+            hb_buffer_close(&in);
+        }
+        if (out != NULL)
+        {
+            hb_fifo_push(filter->fifo_out, out);
+        }
+    }
+}
+
+// Get preview and apply applicable filters
+hb_image_t * hb_get_preview3(hb_handle_t * h, int picture,
+                             hb_dict_t * job_dict)
+{
+    hb_job_t    * job;
+    hb_title_t  * title = NULL;
+    hb_buffer_t * in = NULL, * out;
+
+    job = hb_dict_to_job(h, job_dict);
+    if (job == NULL)
+    {
+        hb_error("hb_get_preview3: failed to unpack job");
+        goto fail;
+    }
+    title = job->title;
+
+    in = hb_read_preview( h, title, picture, HB_PREVIEW_FORMAT_JPG );
+    if (in == NULL)
+    {
+        goto fail;
+    }
+
+    // Initialize supported filters
+    hb_list_t        * list_filter = job->list_filter;
+    hb_filter_init_t   init;
+    int                ii;
+
+    memset(&init, 0, sizeof(init));
+    init.time_base.num = 1;
+    init.time_base.den = 90000;
+    init.job = job;
+    init.pix_fmt = hb_get_best_pix_fmt(job);
+    init.color_range = AVCOL_RANGE_MPEG;
+
+    init.color_prim = title->color_prim;
+    init.color_transfer = title->color_transfer;
+    init.color_matrix = title->color_matrix;
+    init.geometry.width = title->geometry.width;
+    init.geometry.height = title->geometry.height;
+
+    init.geometry.par = job->par;
+    memset(init.crop, 0, sizeof(int[4]));
+    init.vrate = job->vrate;
+    init.cfr = 0;
+    init.grayscale = 0;
+
+    hb_filter_object_t * filter;
+
+    for (ii = 0; ii < hb_list_count(list_filter); )
+    {
+        filter = hb_list_item(list_filter, ii);
+        switch (filter->id)
+        {
+            case HB_FILTER_AVFILTER:
+            case HB_FILTER_CROP_SCALE:
+            case HB_FILTER_PAD:
+            case HB_FILTER_ROTATE:
+            case HB_FILTER_COLORSPACE:
+            case HB_FILTER_DECOMB:
+            case HB_FILTER_DETELECINE:
+            case HB_FILTER_DEINTERLACE:
+            case HB_FILTER_GRAYSCALE:
+                break;
+
+            case HB_FILTER_VFR:
+            case HB_FILTER_RENDER_SUB:
+            case HB_FILTER_QSV:
+            case HB_FILTER_NLMEANS:
+            case HB_FILTER_CHROMA_SMOOTH:
+            case HB_FILTER_LAPSHARP:
+            case HB_FILTER_UNSHARP:
+            case HB_FILTER_DEBLOCK:
+            case HB_FILTER_COMB_DETECT:
+            case HB_FILTER_HQDN3D:
+                // Not implemented, N/A, or requires multiple frame input
+                hb_list_rem(list_filter, filter);
+                hb_filter_close(&filter);
+                continue;
+            default:
+                hb_log("hb_get_preview3: Unrecognized filter (%d)",
+                       filter->id);
+                hb_list_rem(list_filter, filter);
+                hb_filter_close(&filter);
+                continue;
+        }
+        if (filter->init != NULL && filter->init(filter, &init))
+        {
+            hb_error("hb_get_preview3: Failure to initialize filter '%s'",
+                     filter->name);
+            hb_list_rem(list_filter, filter);
+            hb_filter_close(&filter);
+            continue;
+        }
+        ii++;
+    }
+
+    job->pix_fmt = init.pix_fmt;
+    job->color_prim = init.color_prim;
+    job->color_transfer = init.color_transfer;
+    job->color_matrix = init.color_matrix;
+    job->width = init.geometry.width;
+    job->height = init.geometry.height;
+    job->par = init.geometry.par;
+    memcpy(job->crop, init.crop, sizeof(int[4]));
+    job->vrate = init.vrate;
+    job->cfr = init.cfr;
+    job->grayscale = init.grayscale;
+
+    // Add "cropscale"
+    // Adjusts for pixel aspect, performs any requested
+    // post-scaling and sets required pix_fmt AV_PIX_FMT_RGB32
+    //
+    // This will scale the result at the end of the pipeline.
+    // I.e. padding will be scaled
+    hb_rational_t par = init.geometry.par;
+
+    int scaled_width  = init.geometry.width;
+    int scaled_height = init.geometry.height;
+
+    filter = hb_filter_init(HB_FILTER_CROP_SCALE);
+    filter->settings = hb_dict_init();
+    if (par.num >= par.den)
+    {
+        scaled_width = scaled_width * par.num / par.den;
+    }
+    else
+    {
+        scaled_height = scaled_height * par.den / par.num;
+    }
+    hb_dict_set_int(filter->settings, "width", scaled_width);
+    hb_dict_set_int(filter->settings, "height", scaled_height);
+    hb_dict_set_string(filter->settings, "out_pix_fmt", av_get_pix_fmt_name(AV_PIX_FMT_RGB32));
+    hb_list_add(job->list_filter, filter);
+    if (filter->init != NULL && filter->init(filter, &init))
+    {
+        hb_error("hb_get_preview3: Failure to initialize filter '%s'",
+                 filter->name);
+        hb_list_rem(list_filter, filter);
+        hb_filter_close(&filter);
+    }
+
+    hb_avfilter_combine(list_filter);
+
+    for( ii = 0; ii < hb_list_count( list_filter ); )
+    {
+        filter = hb_list_item( list_filter, ii );
+        filter->done = &job->done;
+        if (filter->post_init != NULL && filter->post_init(filter, job))
+        {
+            hb_log( "hb_get_preview3: Failure to initialise filter '%s'",
+                    filter->name );
+            hb_list_rem(list_filter, filter);
+            hb_filter_close(&filter);
+            continue;
+        }
+        ii++;
+    }
+
+    // Set up filter fifos
+    hb_fifo_t *fifo_in, * fifo_first, * fifo_last;
+
+    fifo_in = fifo_first = hb_fifo_init(2, 2);
+    for( ii = 0; ii < hb_list_count( list_filter ); ii++)
+    {
+        filter = hb_list_item(list_filter, ii);
+        if (!filter->skip)
+        {
+            filter->fifo_in = fifo_in;
+            filter->fifo_out = hb_fifo_init(2, 2);
+            fifo_last = fifo_in = filter->fifo_out;
+        }
+    }
+
+    // Feed preview frame to filter chain
+    hb_fifo_push(fifo_first, in);
+    hb_fifo_push(fifo_first, hb_buffer_eof_init());
+
+    // Process the preview frame through all filters
+    for( ii = 0; ii < hb_list_count( list_filter ); ii++)
+    {
+        filter = hb_list_item(list_filter, ii);
+        if (!filter->skip)
+        {
+            process_filter(filter);
+        }
+    }
+    // Retrieve the filtered preview frame
+    out = hb_fifo_get(fifo_last);
+
+    // Close filters
+    for (ii = 0; ii < hb_list_count(list_filter); ii++)
+    {
+        filter = hb_list_item(list_filter, ii);
+        filter->close(filter);
+    }
+
+    // Close fifos
+    hb_fifo_close(&fifo_first);
+    for( ii = 0; ii < hb_list_count( list_filter ); ii++)
+    {
+        filter = hb_list_item(list_filter, ii);
+        hb_fifo_close(&filter->fifo_out);
+    }
+
+    if (out == NULL)
+    {
+        hb_error("hb_get_preview3: Failed to filter preview");
+        goto fail;
+    }
+
+    hb_image_t *image = hb_buffer_to_image(out);
+    hb_buffer_close(&out);
+    if (image->width < 16 || image->height < 16)
+    {
+        // Guard against broken filter generating degenerate images
+        hb_error("hb_get_preview3: bad preview image output by filters");
+        hb_image_close(&image);
+        goto fail;
+    }
+
+    // Clean up
+    hb_job_close(&job);
+
+    return image;
+
+fail:
+
+    {
+        int width = 854, height = 480;
+
+        if (title != NULL)
+        {
+            hb_geometry_t * geo = &title->geometry;
+
+            width = geo->width * geo->par.num / geo->par.den;
+            height = geo->height;
+        }
+
+        image = hb_image_init(AV_PIX_FMT_RGB32, width, height);
+    }
+    hb_job_close(&job);
+
+    return image;
+}
+
  /**
  * Analyzes a frame to detect interlacing artifacts
  * and returns true if interlacing (combing) is found.
diff --git a/libhb/hb_json.c b/libhb/hb_json.c
index 1d83f5e3f..5180f45e2 100644
--- a/libhb/hb_json.c
+++ b/libhb/hb_json.c
@@ -2082,6 +2082,18 @@ char* hb_get_preview_json(hb_handle_t * h, const char *json_param)
     return result;
 }
 
+hb_image_t * hb_get_preview3_json(hb_handle_t * h, int picture, const char *json_job)
+{
+    hb_image_t * image;
+    hb_dict_t  * job_dict;
+
+    job_dict = hb_value_json(json_job);
+    image = hb_get_preview3(h, picture, job_dict);
+    hb_value_free(&job_dict);
+
+    return image;
+}
+
 char* hb_get_preview_params_json(int title_idx, int preview_idx,
                             int deinterlace, hb_geometry_settings_t *settings)
 {
diff --git a/libhb/work.c b/libhb/work.c
index a81700200..d9073e4b5 100644
--- a/libhb/work.c
+++ b/libhb/work.c
@@ -818,103 +818,6 @@ void correct_framerate( hb_interjob_t * interjob, hb_job_t * job )
     }
 }
 
-static int bit_depth_is_supported(hb_job_t * job, int bit_depth)
-{
-    for (int i = 0; i < hb_list_count(job->list_filter); i++)
-    {
-        hb_filter_object_t *filter = hb_list_item(job->list_filter, i);
-
-        switch (filter->id) {
-            case HB_FILTER_DETELECINE:
-            case HB_FILTER_COMB_DETECT:
-            case HB_FILTER_DECOMB:
-            case HB_FILTER_DENOISE:
-            case HB_FILTER_NLMEANS:
-            case HB_FILTER_CHROMA_SMOOTH:
-            case HB_FILTER_LAPSHARP:
-            case HB_FILTER_UNSHARP:
-            case HB_FILTER_GRAYSCALE:
-                return 0;
-        }
-    }
-
-    if (hb_video_encoder_get_depth(job->vcodec) < bit_depth)
-    {
-        return 0;
-    }
-
-    return 1;
-}
-
-static int pix_fmt_is_supported(hb_job_t * job, int pix_fmt)
-{
-    for (int i = 0; i < hb_list_count(job->list_filter); i++)
-    {
-        hb_filter_object_t *filter = hb_list_item(job->list_filter, i);
-
-        switch (filter->id)
-        {
-            case HB_FILTER_DETELECINE:
-            case HB_FILTER_COMB_DETECT:
-            case HB_FILTER_DECOMB:
-            case HB_FILTER_DENOISE:
-            case HB_FILTER_NLMEANS:
-            case HB_FILTER_CHROMA_SMOOTH:
-            case HB_FILTER_LAPSHARP:
-            case HB_FILTER_UNSHARP:
-            case HB_FILTER_GRAYSCALE:
-                if (pix_fmt != AV_PIX_FMT_YUV420P)
-                {
-                    return 0;
-                }
-        }
-    }
-
-    return 1;
-}
-
-static int get_best_pix_ftm(hb_job_t * job)
-{
-    int bit_depth = hb_get_bit_depth(job->title->pix_fmt);
-#if HB_PROJECT_FEATURE_QSV && (defined( _WIN32 ) || defined( __MINGW32__ ))
-    if (hb_qsv_encoder_info_get(hb_qsv_get_adapter_index(), job->vcodec))
-    {
-        if (hb_qsv_full_path_is_enabled(job))
-        {
-            if (job->title->pix_fmt == AV_PIX_FMT_YUV420P10 && job->vcodec == HB_VCODEC_QSV_H265_10BIT)
-            {
-                return AV_PIX_FMT_P010LE;
-            }
-            else
-            {
-                return AV_PIX_FMT_NV12;
-            }
-        }
-        else
-        {
-            // system memory usage: QSV encoder only or QSV decoder + SW filters + QSV encoder
-            return AV_PIX_FMT_YUV420P;
-        }
-    }
-#endif
-    if (job->vcodec == HB_VCODEC_FFMPEG_MF_H264 || job->vcodec == HB_VCODEC_FFMPEG_MF_H265)
-    {
-        if (pix_fmt_is_supported(job, AV_PIX_FMT_NV12))
-        {
-            return AV_PIX_FMT_NV12;
-        }
-    }
-    if (bit_depth >= 12 && bit_depth_is_supported(job, 12))
-    {
-        return AV_PIX_FMT_YUV420P12;
-    }
-    if (bit_depth >= 10 && bit_depth_is_supported(job, 10))
-    {
-        return AV_PIX_FMT_YUV420P10;
-    }
-    return AV_PIX_FMT_YUV420P;
-}
-
 static void analyze_subtitle_scan( hb_job_t * job )
 {
     hb_subtitle_t *subtitle;
@@ -1531,7 +1434,7 @@ static void do_job(hb_job_t *job)
         init.time_base.num = 1;
         init.time_base.den = 90000;
         init.job = job;
-        init.pix_fmt = get_best_pix_ftm(job);
+        init.pix_fmt = hb_get_best_pix_fmt(job);
         init.color_range = AVCOL_RANGE_MPEG;
 
         init.color_prim = title->color_prim;