hb_ff_set_sample_fmt() improvements.

- allow setting a sample_fmt other than AV_SAMPLE_FMT_FLT

- fall back on planar/packed variant of the requested format if available



git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@5019 b64f7644-9d1e-0410-96f1-a4d463321fa5

5 files changed, 48 insertions, 24 deletions

diff --git a/libhb/decavcodec.c b/libhb/decavcodec.c
index 984b1cf87..4fd22c833 100644
--- a/libhb/decavcodec.c
+++ b/libhb/decavcodec.c
@@ -205,14 +205,14 @@ static int decavcodecaInit( hb_work_object_t * w, hb_job_t * job )
         AVFormatContext *ic = (AVFormatContext*)pv->title->opaque_priv;
         pv->context = avcodec_alloc_context3(codec);
         avcodec_copy_context( pv->context, ic->streams[w->audio->id]->codec);
-        hb_ff_set_sample_fmt( pv->context, codec );
+        hb_ff_set_sample_fmt( pv->context, codec, AV_SAMPLE_FMT_FLT );
     }
     else
     {
         pv->parser = av_parser_init( w->codec_param );
 
         pv->context = avcodec_alloc_context3(codec);
-        hb_ff_set_sample_fmt( pv->context, codec );
+        hb_ff_set_sample_fmt( pv->context, codec, AV_SAMPLE_FMT_FLT );
     }
     if ( hb_avcodec_open( pv->context, codec, NULL, 0 ) )
     {
@@ -393,7 +393,7 @@ static int decavcodecaBSInfo( hb_work_object_t *w, const hb_buffer_t *buf,
 
     AVCodecParserContext *parser = av_parser_init( codec->id );
     AVCodecContext *context = avcodec_alloc_context3(codec);
-    hb_ff_set_sample_fmt( context, codec );
+    hb_ff_set_sample_fmt( context, codec, AV_SAMPLE_FMT_FLT );
     if ( hb_avcodec_open( context, codec, NULL, 0 ) )
     {
         return -1;
diff --git a/libhb/encavcodecaudio.c b/libhb/encavcodecaudio.c
index fe398ab4f..0fc23faa0 100644
--- a/libhb/encavcodecaudio.c
+++ b/libhb/encavcodecaudio.c
@@ -89,8 +89,15 @@ static int encavcodecaInit(hb_work_object_t *w, hb_job_t *job)
         context->compression_level = audio->config.out.compression_level;
     }
 
-    // Try to set format to float; fall back to whatever is supported.
-    hb_ff_set_sample_fmt(context, codec);
+    // set the sample_fmt to something practical
+    if (audio->config.out.codec == HB_ACODEC_FFFLAC)
+    {
+        hb_ff_set_sample_fmt(context, codec, AV_SAMPLE_FMT_S16);
+    }
+    else
+    {
+        hb_ff_set_sample_fmt(context, codec, AV_SAMPLE_FMT_FLT);
+    }
 
     if (hb_avcodec_open(context, codec, &av_opts, 0))
     {
diff --git a/libhb/hb.c b/libhb/hb.c
index 12377a8b5..52ddc206b 100644
--- a/libhb/hb.c
+++ b/libhb/hb.c
@@ -285,30 +285,47 @@ uint64_t hb_ff_layout_xlat(uint64_t ff_channel_layout, int nchannels)
     return hb_layout;
 }
 
-// Set sample format to AV_SAMPLE_FMT_FLT if supported.
-// If it is not supported, we will have to translate using
-// av_audio_convert.
-void hb_ff_set_sample_fmt(AVCodecContext *context, AVCodec *codec)
+/*
+ * Set sample format to the request format if supported by the codec.
+ * The planar/packed variant of the requested format is the next best thing.
+ */
+void hb_ff_set_sample_fmt(AVCodecContext *context, AVCodec *codec,
+                          enum AVSampleFormat request_sample_fmt)
 {
-    if ( codec && codec->sample_fmts )
+    if (context != NULL && codec != NULL &&
+        codec->type == AVMEDIA_TYPE_AUDIO && codec->sample_fmts != NULL)
     {
-        if ( codec->type != AVMEDIA_TYPE_AUDIO )
-            return; // Not audio
-
         const enum AVSampleFormat *fmt;
+        enum AVSampleFormat next_best_fmt;
+
+        next_best_fmt = (av_sample_fmt_is_planar(request_sample_fmt)  ?
+                         av_get_packed_sample_fmt(request_sample_fmt) :
+                         av_get_planar_sample_fmt(request_sample_fmt));
 
-        for ( fmt = codec->sample_fmts; *fmt != -1; fmt++ )
+        context->request_sample_fmt = AV_SAMPLE_FMT_NONE;
+
+        for (fmt = codec->sample_fmts; *fmt != AV_SAMPLE_FMT_NONE; fmt++)
         {
-            if ( *fmt == AV_SAMPLE_FMT_FLT )
+            if (*fmt == request_sample_fmt)
             {
-                context->request_sample_fmt = AV_SAMPLE_FMT_FLT;
-                context->sample_fmt = AV_SAMPLE_FMT_FLT;
+                context->request_sample_fmt = request_sample_fmt;
                 break;
             }
+            else if (*fmt == next_best_fmt)
+            {
+                context->request_sample_fmt = next_best_fmt;
+            }
         }
-        if ( *fmt == -1 )
-            context->sample_fmt = codec->sample_fmts[0];
 
+        /*
+         * When encoding and AVCodec.sample_fmts exists, avcodec_open2()
+         * will error out if AVCodecContext.sample_fmt isn't set.
+         */
+        if (context->request_sample_fmt == AV_SAMPLE_FMT_NONE)
+        {
+            context->request_sample_fmt = codec->sample_fmts[0];
+        }
+        context->sample_fmt = context->request_sample_fmt;
     }
 }
 
diff --git a/libhb/hbffmpeg.h b/libhb/hbffmpeg.h
index 51b51d8af..2de771505 100644
--- a/libhb/hbffmpeg.h
+++ b/libhb/hbffmpeg.h
@@ -24,7 +24,7 @@ int  hb_avcodec_close(AVCodecContext *);
 
 uint64_t hb_ff_layout_xlat(uint64_t ff_channel_layout, int nchannels);
 uint64_t hb_ff_mixdown_xlat(int hb_mixdown, int *downmix_mode);
-void     hb_ff_set_sample_fmt(AVCodecContext *context, AVCodec *codec);
+void     hb_ff_set_sample_fmt(AVCodecContext *, AVCodec *, enum AVSampleFormat);
 
 struct SwsContext*
 hb_sws_get_context(int srcW, int srcH, enum PixelFormat srcFormat,
diff --git a/libhb/sync.c b/libhb/sync.c
index cd45b7fdd..c3e4c37a1 100644
--- a/libhb/sync.c
+++ b/libhb/sync.c
@@ -946,12 +946,12 @@ static void InitAudio( hb_job_t * job, hb_sync_common_t * common, int i )
             } break;
         }
 
-        c     = avcodec_alloc_context3( codec );
-
+        c              = avcodec_alloc_context3(codec);
         c->bit_rate    = w->audio->config.in.bitrate;
         c->sample_rate = w->audio->config.in.samplerate;
-        c->channels    = av_get_channel_layout_nb_channels(w->audio->config.in.channel_layout);
-        hb_ff_set_sample_fmt( c, codec );
+        c->channels    =
+            av_get_channel_layout_nb_channels(w->audio->config.in.channel_layout);
+        hb_ff_set_sample_fmt(c, codec, AV_SAMPLE_FMT_FLT);
 
         if (w->audio->config.in.channel_layout == AV_CH_LAYOUT_STEREO_DOWNMIX)
         {