Add support for VCE hardware encoding.

6 files changed, 160 insertions, 10 deletions

diff --git a/libhb/common.c b/libhb/common.c
index cc10238e1..9c1c42f32 100644
--- a/libhb/common.c
+++ b/libhb/common.c
@@ -236,12 +236,14 @@ hb_encoder_internal_t hb_video_encoders[]  =
     { { "H.264 (x264)",        "x264",       "H.264 (libx264)",         HB_VCODEC_X264_8BIT,         HB_MUX_MASK_MP4|HB_MUX_MASK_MKV, }, NULL, 1, HB_GID_VCODEC_H264,   },
     { { "H.264 10-bit (x264)", "x264_10bit", "H.264 10-bit (libx264)",  HB_VCODEC_X264_10BIT,   HB_MUX_MASK_MP4|HB_MUX_MASK_MKV, }, NULL, 1, HB_GID_VCODEC_H264,   },
     { { "H.264 (Intel QSV)",   "qsv_h264",   "H.264 (Intel Media SDK)", HB_VCODEC_QSV_H264,     HB_MUX_MASK_MP4|HB_MUX_MASK_MKV, }, NULL, 1, HB_GID_VCODEC_H264,   },
+    { { "H.264 (AMD VCE)",     "vce_h264",   "H.264 (libavcodec)",      HB_VCODEC_FFMPEG_VCE_H264,     HB_MUX_MASK_MP4|HB_MUX_MASK_MKV, }, NULL, 1, HB_GID_VCODEC_H264,   },
     { { "H.265 (x265)",        "x265",       "H.265 (libx265)",         HB_VCODEC_X265_8BIT,      HB_MUX_AV_MP4|HB_MUX_AV_MKV,   }, NULL, 1, HB_GID_VCODEC_H265,   },
     { { "H.265 10-bit (x265)", "x265_10bit", "H.265 10-bit (libx265)",  HB_VCODEC_X265_10BIT,     HB_MUX_AV_MP4|HB_MUX_AV_MKV,   }, NULL, 1, HB_GID_VCODEC_H265,   },
     { { "H.265 12-bit (x265)", "x265_12bit", "H.265 12-bit (libx265)",  HB_VCODEC_X265_12BIT,     HB_MUX_AV_MP4|HB_MUX_AV_MKV,   }, NULL, 1, HB_GID_VCODEC_H265,   },
     { { "H.265 16-bit (x265)", "x265_16bit", "H.265 16-bit (libx265)",  HB_VCODEC_X265_16BIT,     HB_MUX_AV_MP4|HB_MUX_AV_MKV,   }, NULL, 1, HB_GID_VCODEC_H265,   },
     { { "H.265 (Intel QSV)",   "qsv_h265",   "H.265 (Intel Media SDK)", HB_VCODEC_QSV_H265,     HB_MUX_MASK_MP4|HB_MUX_MASK_MKV, }, NULL, 1, HB_GID_VCODEC_H265,   },
     { { "H.265 10-bit (Intel QSV)",   "qsv_h265_10bit",   "H.265 10-bit (Intel Media SDK)", HB_VCODEC_QSV_H265_10BIT,     HB_MUX_MASK_MP4|HB_MUX_MASK_MKV, }, NULL, 1, HB_GID_VCODEC_H265,   },
+    { { "H.265 (AMD VCE)",     "vce_h265",   "H.265 (libavcodec)",      HB_VCODEC_FFMPEG_VCE_H265,     HB_MUX_MASK_MP4|HB_MUX_MASK_MKV, }, NULL, 1, HB_GID_VCODEC_H264,   },
     { { "MPEG-4",              "mpeg4",      "MPEG-4 (libavcodec)",     HB_VCODEC_FFMPEG_MPEG4, HB_MUX_MASK_MP4|HB_MUX_MASK_MKV, }, NULL, 1, HB_GID_VCODEC_MPEG4,  },
     { { "MPEG-2",              "mpeg2",      "MPEG-2 (libavcodec)",     HB_VCODEC_FFMPEG_MPEG2, HB_MUX_MASK_MP4|HB_MUX_MASK_MKV, }, NULL, 1, HB_GID_VCODEC_MPEG2,  },
     { { "VP8",                 "VP8",        "VP8 (libvpx)",            HB_VCODEC_FFMPEG_VP8,                   HB_MUX_MASK_MKV, }, NULL, 1, HB_GID_VCODEC_VP8,    },
@@ -265,6 +267,8 @@ static int hb_video_encoder_is_enabled(int encoder)
         case HB_VCODEC_FFMPEG_MPEG2:
         case HB_VCODEC_FFMPEG_VP8:
         case HB_VCODEC_FFMPEG_VP9:
+        case HB_VCODEC_FFMPEG_VCE_H264:
+        case HB_VCODEC_FFMPEG_VCE_H265:
             return 1;
 
 #ifdef USE_X265
@@ -1472,6 +1476,11 @@ const char* const* hb_video_encoder_get_profiles(int encoder)
         case HB_VCODEC_X265_16BIT:
             return hb_h265_profile_names_16bit;
 
+        case HB_VCODEC_FFMPEG_VCE_H264:
+            return hb_h264_profile_names_8bit;
+        case HB_VCODEC_FFMPEG_VCE_H265:
+            return hb_h265_profile_names_8bit;
+
         default:
             return NULL;
     }
diff --git a/libhb/common.h b/libhb/common.h
index 370ae5f55..27dad4e6b 100644
--- a/libhb/common.h
+++ b/libhb/common.h
@@ -501,14 +501,16 @@ struct hb_job_s
          cfr:               0 (vfr), 1 (cfr), 2 (pfr) [see render.c]
          pass:              0, 1 or 2 (or -1 for scan)
          areBframes:        boolean to note if b-frames are used */
-#define HB_VCODEC_MASK         0x00FFFFF
+#define HB_VCODEC_MASK         0x08FFFFF
 #define HB_VCODEC_INVALID      0x0000000
 #define HB_VCODEC_THEORA       0x0000002
 #define HB_VCODEC_FFMPEG_MPEG4 0x0000010
 #define HB_VCODEC_FFMPEG_MPEG2 0x0000020
 #define HB_VCODEC_FFMPEG_VP8   0x0000040
 #define HB_VCODEC_FFMPEG_VP9   0x0000080
-#define HB_VCODEC_FFMPEG_MASK  0x00000F0
+#define HB_VCODEC_FFMPEG_VCE_H264 0x00040000
+#define HB_VCODEC_FFMPEG_VCE_H265 0x00080000
+#define HB_VCODEC_FFMPEG_MASK  (0x00000F0|HB_VCODEC_FFMPEG_VCE_H264|HB_VCODEC_FFMPEG_VCE_H265)
 #define HB_VCODEC_QSV_H264     0x0000100
 #define HB_VCODEC_QSV_H265_8BIT     0x0000200
 #define HB_VCODEC_QSV_H265_10BIT    0x0000400
@@ -519,14 +521,14 @@ struct hb_job_s
 #define HB_VCODEC_X264         HB_VCODEC_X264_8BIT
 #define HB_VCODEC_X264_10BIT   0x0020000
 #define HB_VCODEC_X264_MASK    0x0030000
-#define HB_VCODEC_H264_MASK    (HB_VCODEC_X264_MASK|HB_VCODEC_QSV_H264)
+#define HB_VCODEC_H264_MASK    (HB_VCODEC_X264_MASK|HB_VCODEC_QSV_H264|HB_VCODEC_FFMPEG_VCE_H264)
 #define HB_VCODEC_X265_8BIT    0x0001000
 #define HB_VCODEC_X265         HB_VCODEC_X265_8BIT
 #define HB_VCODEC_X265_10BIT   0x0002000
 #define HB_VCODEC_X265_12BIT   0x0004000
 #define HB_VCODEC_X265_16BIT   0x0008000
 #define HB_VCODEC_X265_MASK    0x000F000
-#define HB_VCODEC_H265_MASK    (HB_VCODEC_X265_MASK|HB_VCODEC_QSV_H265_MASK)
+#define HB_VCODEC_H265_MASK    (HB_VCODEC_X265_MASK|HB_VCODEC_QSV_H265_MASK|HB_VCODEC_FFMPEG_VCE_H265)
 
 /* define an invalid CQ value compatible with all CQ-capable codecs */
 #define HB_INVALID_VIDEO_QUALITY (-1000.)
diff --git a/libhb/encavcodec.c b/libhb/encavcodec.c
index fa5d27031..fc78039a5 100644
--- a/libhb/encavcodec.c
+++ b/libhb/encavcodec.c
@@ -10,6 +10,9 @@
 #include "hb.h"
 #include "hb_dict.h"
 #include "hbffmpeg.h"
+#include "h264_common.h"
+#include "h265_common.h"
+#include "nal_units.h"
 
 /*
  * The frame info struct remembers information about each frame across calls
@@ -88,19 +91,33 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
         case AV_CODEC_ID_MPEG4:
         {
             hb_log("encavcodecInit: MPEG-4 ASP encoder");
+            codec = avcodec_find_encoder_by_name("mpeg4");
         } break;
         case AV_CODEC_ID_MPEG2VIDEO:
         {
             hb_log("encavcodecInit: MPEG-2 encoder");
+            codec = avcodec_find_encoder_by_name("mpeg2video");
         } break;
         case AV_CODEC_ID_VP8:
         {
             hb_log("encavcodecInit: VP8 encoder");
+            codec = avcodec_find_encoder_by_name("libvpx_vp8");
         } break;
         case AV_CODEC_ID_VP9:
         {
             hb_log("encavcodecInit: VP9 encoder");
+            codec = avcodec_find_encoder_by_name("libvpx_vp9");
         } break;
+        case AV_CODEC_ID_H264:
+        {
+            hb_log("encavcodecInit: H.264 (AMD VCE)");
+            codec = avcodec_find_encoder_by_name("h264_amf");
+        }break;
+        case AV_CODEC_ID_HEVC:
+        {
+            hb_log("encavcodecInit: H.265 (AMD VCE)");
+            codec = avcodec_find_encoder_by_name("hevc_amf");
+        }break;
         default:
         {
             hb_error("encavcodecInit: unsupported encoder!");
@@ -109,7 +126,6 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
         }
     }
 
-    codec = avcodec_find_encoder( w->codec_param  );
     if( !codec )
     {
         hb_log( "encavcodecInit: avcodec_find_encoder "
@@ -175,6 +191,20 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
     context->time_base.num = fps.den;
     context->gop_size  = ((double)job->orig_vrate.num / job->orig_vrate.den +
                                   0.5) * 10;
+    if ((job->vcodec == HB_VCODEC_FFMPEG_VCE_H264) || (job->vcodec == HB_VCODEC_FFMPEG_VCE_H265))
+    {
+        // Set encoder preset
+        context->profile = FF_PROFILE_UNKNOWN;
+        if (job->encoder_preset != NULL && *job->encoder_preset)
+        {
+            if ((!strcasecmp(job->encoder_preset, "balanced"))
+                || (!strcasecmp(job->encoder_preset, "speed"))
+                || (!strcasecmp(job->encoder_preset, "quality")))
+            {
+                av_opt_set(context, "quality", job->encoder_preset, AV_OPT_SEARCH_CHILDREN);
+            }
+        }
+    }
 
     /* place job->encoder_options in an hb_dict_t for convenience */
     hb_dict_t * lavc_opts = NULL;
@@ -285,6 +315,56 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
         context->flags |= AV_CODEC_FLAG_GRAY;
     }
 
+    if (job->vcodec == HB_VCODEC_FFMPEG_VCE_H264)
+    {
+        // Set profile and level
+        context->profile = FF_PROFILE_UNKNOWN;
+        if (job->encoder_profile != NULL && *job->encoder_profile)
+        {
+            if (!strcasecmp(job->encoder_profile, "baseline"))
+                context->profile = FF_PROFILE_H264_BASELINE;
+            else if (!strcasecmp(job->encoder_profile, "main"))
+                 context->profile = FF_PROFILE_H264_MAIN;
+            else if (!strcasecmp(job->encoder_profile, "high"))
+                context->profile = FF_PROFILE_H264_HIGH;
+        }
+        context->level = FF_LEVEL_UNKNOWN;
+        if (job->encoder_level != NULL && *job->encoder_level)
+        {
+            int i = 1;
+            while (hb_h264_level_names[i] != NULL)
+            {
+                if (!strcasecmp(job->encoder_level, hb_h264_level_names[i]))
+                    context->level = hb_h264_level_values[i];
+                ++i;
+            }
+        }
+    }
+
+    if (job->vcodec == HB_VCODEC_FFMPEG_VCE_H265)
+    {
+        // Set profile and level
+        context->profile = FF_PROFILE_UNKNOWN;
+        if (job->encoder_profile != NULL && *job->encoder_profile)
+        {
+            if (!strcasecmp(job->encoder_profile, "main"))
+                 context->profile = FF_PROFILE_HEVC_MAIN;
+        }
+        context->level = FF_LEVEL_UNKNOWN;
+        if (job->encoder_level != NULL && *job->encoder_level)
+        {
+            int i = 1;
+            while (hb_h265_level_names[i] != NULL)
+            {
+                if (!strcasecmp(job->encoder_level, hb_h265_level_names[i]))
+                    context->level = hb_h265_level_values[i];
+                ++i;
+            }
+        }
+        // FIXME
+        //context->tier = FF_TIER_UNKNOWN;
+    }
+
     if( job->pass_id == HB_PASS_ENCODE_1ST ||
         job->pass_id == HB_PASS_ENCODE_2ND )
     {
@@ -349,6 +429,7 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
     if (hb_avcodec_open(context, codec, &av_opts, HB_FFMPEG_THREADS_AUTO))
     {
         hb_log( "encavcodecInit: avcodec_open failed" );
+        return 1;
     }
 
     if (job->pass_id == HB_PASS_ENCODE_1ST &&
@@ -374,11 +455,36 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
     {
         job->areBframes = 1;
     }
+
     if( ( job->mux & HB_MUX_MASK_MP4 ) && job->pass_id != HB_PASS_ENCODE_1ST )
     {
-        w->config->mpeg4.length = context->extradata_size;
-        memcpy( w->config->mpeg4.bytes, context->extradata,
-                context->extradata_size );
+        if (w->codec_param == AV_CODEC_ID_H264) // FIXME: h265 as well?
+        {
+            // Scan extradata for the SPS/PPS headers
+            unsigned char *data = context->extradata;
+            unsigned char *dataEnd = context->extradata + context->extradata_size;
+            size_t len = dataEnd - data;
+
+            while ((data = hb_annexb_find_next_nalu(data, &len)) != NULL) {
+                if ((data[0] & 0x1f) == 7) {
+                    // SPS found, copy into work object
+                    w->config->h264.sps_length = len;
+                    memcpy(w->config->h264.sps, data, len);
+                }
+                if ((data[0] & 0x1f) == 8) {
+                    // PPS found, copy into work object
+                    w->config->h264.pps_length = len;
+                    memcpy(w->config->h264.pps, data, len);
+                }
+                len = dataEnd - data;
+            } 
+        }
+        else
+        {
+            w->config->mpeg4.length = context->extradata_size;
+            memcpy( w->config->mpeg4.bytes, context->extradata,
+                    context->extradata_size );
+        }
     }
 
 done:
@@ -507,6 +613,7 @@ static hb_buffer_t * process_delay_list( hb_work_private_t * pv, hb_buffer_t * b
 static void get_packets( hb_work_object_t * w, hb_buffer_list_t * list )
 {
     hb_work_private_t * pv = w->private_data;
+    hb_job_t * job = pv->job;
 
     while (1)
     {
@@ -524,8 +631,16 @@ static void get_packets( hb_work_object_t * w, hb_buffer_list_t * list )
         {
             hb_log("encavcodec: avcodec_receive_packet failed");
         }
-        out = hb_buffer_init(pkt.size);
-        memcpy(out->data, pkt.data, out->size);
+
+        if (job->vcodec == HB_VCODEC_FFMPEG_VCE_H264)
+        {
+            out = hb_nal_bitstream_annexb_to_mp4(pkt.data, pkt.size);
+        }
+        else
+        {
+            out = hb_buffer_init(pkt.size);
+            memcpy(out->data, pkt.data, out->size);
+        }
 
         int64_t frameno = pkt.pts;
         out->size       = pkt.size;
@@ -736,6 +851,10 @@ const char* const* hb_av_preset_get_names(int encoder)
         case HB_VCODEC_FFMPEG_VP9:
             return vpx_preset_names;
 
+        case HB_VCODEC_FFMPEG_VCE_H264:
+        case HB_VCODEC_FFMPEG_VCE_H265:
+            return hb_vce_preset_names;
+
         default:
             return NULL;
     }
diff --git a/libhb/hbffmpeg.h b/libhb/hbffmpeg.h
index 890b62ffc..4ed2ef593 100644
--- a/libhb/hbffmpeg.h
+++ b/libhb/hbffmpeg.h
@@ -7,6 +7,9 @@
    For full terms see the file COPYING file or visit http://www.gnu.org/licenses/gpl-2.0.html
  */
 
+#ifndef HB_FFMPEG_H
+#define HB_FFMPEG_H
+
 #include "libavcodec/avcodec.h"
 #include "libavformat/avformat.h"
 #include "libavutil/channel_layout.h"
@@ -37,6 +40,9 @@ hb_sws_get_context(int srcW, int srcH, enum AVPixelFormat srcFormat,
                    int dstW, int dstH, enum AVPixelFormat dstFormat,
                    int flags, int colorspace);
 
+static const char* const hb_vce_preset_names[] = { "speed", "balanced", "quality", NULL, };
+
 hb_buffer_t * hb_avframe_to_video_buffer(AVFrame *frame, AVRational time_base);
 void hb_avframe_set_video_buffer_flags(hb_buffer_t * buf, AVFrame *frame,
                                        AVRational time_base);
+#endif
diff --git a/libhb/muxavformat.c b/libhb/muxavformat.c
index 698096600..c648e7ca2 100644
--- a/libhb/muxavformat.c
+++ b/libhb/muxavformat.c
@@ -214,6 +214,7 @@ static int avformatInit( hb_mux_object_t * m )
         case HB_VCODEC_X264_8BIT:
         case HB_VCODEC_X264_10BIT:
         case HB_VCODEC_QSV_H264:
+        case HB_VCODEC_FFMPEG_VCE_H264:
             track->st->codecpar->codec_id = AV_CODEC_ID_H264;
             if (job->mux == HB_MUX_AV_MP4 && job->inline_parameter_sets)
             {
@@ -339,6 +340,7 @@ static int avformatInit( hb_mux_object_t * m )
         case HB_VCODEC_X265_16BIT:
         case HB_VCODEC_QSV_H265:
         case HB_VCODEC_QSV_H265_10BIT:
+        case HB_VCODEC_FFMPEG_VCE_H265:
             track->st->codecpar->codec_id  = AV_CODEC_ID_HEVC;
             if (job->mux == HB_MUX_AV_MP4 && job->inline_parameter_sets)
             {
diff --git a/libhb/work.c b/libhb/work.c
index c2eeb34e6..19d78b2e4 100644
--- a/libhb/work.c
+++ b/libhb/work.c
@@ -243,6 +243,14 @@ hb_work_object_t* hb_video_encoder(hb_handle_t *h, int vcodec)
             w = hb_get_work(h, WORK_ENCX265);
             break;
 #endif
+        case HB_VCODEC_FFMPEG_VCE_H264:
+            w = hb_get_work(h, WORK_ENCAVCODEC);
+            w->codec_param = AV_CODEC_ID_H264;
+            break;
+        case HB_VCODEC_FFMPEG_VCE_H265:
+            w = hb_get_work(h, WORK_ENCAVCODEC);
+            w->codec_param = AV_CODEC_ID_HEVC;
+            break;
         default:
             hb_error("Unknown video codec (0x%x)", vcodec );
     }
@@ -474,6 +482,8 @@ void hb_display_job_info(hb_job_t *job)
                 case HB_VCODEC_QSV_H264:
                 case HB_VCODEC_QSV_H265:
                 case HB_VCODEC_QSV_H265_10BIT:
+                case HB_VCODEC_FFMPEG_VCE_H264:
+                case HB_VCODEC_FFMPEG_VCE_H265:
                     hb_log("     + profile: %s", job->encoder_profile);
                 default:
                     break;
@@ -488,6 +498,8 @@ void hb_display_job_info(hb_job_t *job)
                 case HB_VCODEC_QSV_H264:
                 case HB_VCODEC_QSV_H265:
                 case HB_VCODEC_QSV_H265_10BIT:
+                case HB_VCODEC_FFMPEG_VCE_H264:
+                case HB_VCODEC_FFMPEG_VCE_H265:
                     hb_log("     + level:   %s", job->encoder_level);
                 default:
                     break;