Add NVEnc encoder. (Round 3) (#1437)

Adding the Nvidia NVEnc H.264 and H.265 encoders.
Based on Initial work by sgothel

--enable-nvenc is the new compile time configure option to enable for builds.

1 files changed, 100 insertions, 9 deletions

diff --git a/libhb/encavcodec.c b/libhb/encavcodec.c
index ee914faba..17a6fdb25 100644
--- a/libhb/encavcodec.c
+++ b/libhb/encavcodec.c
@@ -68,6 +68,22 @@ static const char * const vpx_preset_names[] =
     "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow", NULL
 };
 
+static const char * const h26x_nvenc_preset_names[] =
+{
+    "losslesshp", "lossless", "llhp", "llhq", "ll", "bd", "hq", "hp", "fast", "medium", "slow", "default", NULL
+};
+
+static const char * const h264_nvenc_profile_names[] =
+{
+    "auto", "baseline", "main", "high", NULL  // "high444p" not supported. 
+};
+
+static const char * const h265_nvenc_profile_names[] =
+{
+    "auto", "main", "main10", "rext", NULL 
+};
+
+
 int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
 {
     int ret = 0;
@@ -75,6 +91,7 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
     AVCodec * codec;
     AVCodecContext * context;
     AVRational fps;
+    int64_t bit_rate_ceiling = -1;
 
     hb_work_private_t * pv = calloc( 1, sizeof( hb_work_private_t ) );
     w->private_data   = pv;
@@ -110,13 +127,29 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
         } break;
         case AV_CODEC_ID_H264:
         {
-            hb_log("encavcodecInit: H.264 (AMD VCE)");
-            codec = avcodec_find_encoder_by_name("h264_amf");
+            switch (job->vcodec) {
+                case HB_VCODEC_FFMPEG_NVENC_H264:
+                    hb_log("encavcodecInit: H.264 (Nvidia NVENC)");
+                    codec = avcodec_find_encoder_by_name("h264_nvenc");
+                    break;
+                case HB_VCODEC_FFMPEG_VCE_H264:
+                    hb_log("encavcodecInit: H.264 (AMD VCE)");
+                    codec = avcodec_find_encoder_by_name("h264_amf");
+                    break;
+            }
         }break;
         case AV_CODEC_ID_HEVC:
         {
-            hb_log("encavcodecInit: H.265 (AMD VCE)");
-            codec = avcodec_find_encoder_by_name("hevc_amf");
+            switch (job->vcodec) {
+                case HB_VCODEC_FFMPEG_NVENC_H265:
+                    hb_log("encavcodecInit: H.265 (Nvidia NVENC)");
+                    codec = avcodec_find_encoder_by_name("hevc_nvenc");
+                    break;
+                case HB_VCODEC_FFMPEG_VCE_H264:
+                    hb_log("encavcodecInit: H.265 (AMD VCE)");
+                    codec = avcodec_find_encoder_by_name("hevc_amf");
+                    break;
+            }
         }break;
         default:
         {
@@ -213,6 +246,8 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
         lavc_opts = hb_encopts_to_dict(job->encoder_options, job->vcodec);
     }
 
+    bit_rate_ceiling = (int64_t)job->width * (int64_t)job->height * (int64_t)fps.num / (int64_t)fps.den;
+ 
     if (job->vquality != HB_INVALID_VIDEO_QUALITY)
     {
         if ( w->codec_param == AV_CODEC_ID_VP8 ||
@@ -221,7 +256,7 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
             //This value was chosen to make the bitrate high enough
             //for libvpx to "turn off" the maximum bitrate feature
             //that is normally applied to constant quality.
-            context->bit_rate = job->width * job->height * fps.num / fps.den;
+            context->bit_rate = bit_rate_ceiling;
         }
     }
 
@@ -263,14 +298,16 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
     else
     {
         /* Constant quantizer */
-        // These settings produce better image quality than
-        // what was previously used
-        context->flags |= AV_CODEC_FLAG_QSCALE;
-        context->global_quality = FF_QP2LAMBDA * job->vquality + 0.5;
+
         //Set constant quality for libvpx
         if ( w->codec_param == AV_CODEC_ID_VP8 ||
              w->codec_param == AV_CODEC_ID_VP9 )
         {
+            // These settings produce better image quality than
+            // what was previously used
+            context->flags |= AV_CODEC_FLAG_QSCALE;
+            context->global_quality = FF_QP2LAMBDA * job->vquality + 0.5;
+        
             char quality[7];
             snprintf(quality, 7, "%.2f", job->vquality);
             av_dict_set( &av_opts, "crf", quality, 0 );
@@ -280,8 +317,40 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
             context->bit_rate = job->width * job->height * fps.num / fps.den;
             hb_log( "encavcodec: encoding at CQ %.2f", job->vquality );
         }
+        //Set constant quality for nvenc
+        else if ( job->vcodec == HB_VCODEC_FFMPEG_NVENC_H264 ||
+                  job->vcodec == HB_VCODEC_FFMPEG_NVENC_H265 )
+        {
+            char quality[7];
+            snprintf(quality, 7, "%.2f", job->vquality);
+            av_dict_set( &av_opts, "rc", "vbr", 0 );
+            av_dict_set( &av_opts, "cq", quality, 0 );
+
+            // further Advanced Quality Settings in Constant Quality Mode
+            av_dict_set( &av_opts, "init_qpP", "1", 0 );
+            av_dict_set( &av_opts, "init_qpB", "1", 0 );
+            av_dict_set( &av_opts, "init_qpI", "1", 0 );
+            av_dict_set( &av_opts, "rc-lookahead", "16", 0 ); // also adds b-frames (h264 only it seems for now), max 32 causes errors
+            if( job->vcodec == HB_VCODEC_FFMPEG_NVENC_H265 ) {
+                av_dict_set( &av_opts, "spatial_aq", "1", 0 ); // oops, nvenc_hevc.c uses an underscore
+            } else {
+                av_dict_set( &av_opts, "spatial-aq", "1", 0 ); // oops, nvenc_h264.c uses a dash
+            }
+            hb_log( "encavcodec: encoding at rc=vbr CQ %.2f, init_qp 1, rc-lookahead 16, spatial_aq 1, aq-strength default", job->vquality );
+
+            //This value was chosen to make the bitrate high enough
+            //for nvenc to "turn off" the maximum bitrate feature
+            //that is normally applied to constant quality.
+            context->bit_rate = bit_rate_ceiling;
+            hb_log( "encavcodec: bit_rate.4 %ld", context->bit_rate);
+        }
         else
         {
+            // These settings produce better image quality than
+            // what was previously used
+            context->flags |= AV_CODEC_FLAG_QSCALE;
+            context->global_quality = FF_QP2LAMBDA * job->vquality + 0.5;
+            
             hb_log( "encavcodec: encoding at constant quantizer %d",
                     context->global_quality );
         }
@@ -841,6 +910,10 @@ static int apply_encoder_preset(int vcodec, AVDictionary ** av_opts,
         case HB_VCODEC_FFMPEG_VP8:
         case HB_VCODEC_FFMPEG_VP9:
             return apply_vpx_preset(av_opts, preset);
+        case HB_VCODEC_FFMPEG_NVENC_H264:
+        case HB_VCODEC_FFMPEG_NVENC_H265:
+             av_dict_set( av_opts, "preset", preset, 0);
+             break;
         default:
             break;
     }
@@ -860,7 +933,25 @@ const char* const* hb_av_preset_get_names(int encoder)
         case HB_VCODEC_FFMPEG_VCE_H265:
             return hb_vce_preset_names;
 
+        case HB_VCODEC_FFMPEG_NVENC_H264:
+        case HB_VCODEC_FFMPEG_NVENC_H265:
+            return h26x_nvenc_preset_names;
+
         default:
             return NULL;
     }
 }
+
+const char* const* hb_av_profile_get_names(int encoder)
+{
+    switch (encoder)
+    {
+        case HB_VCODEC_FFMPEG_NVENC_H264:
+            return h264_nvenc_profile_names;
+        case HB_VCODEC_FFMPEG_NVENC_H265:
+            return h265_nvenc_profile_names;
+
+         default:
+             return NULL;
+     }
+}