hb_audio_remap: protect against invalid channel layouts.

git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@5074 b64f7644-9d1e-0410-96f1-a4d463321fa5

4 files changed, 18 insertions, 7 deletions

diff --git a/libhb/audio_remap.c b/libhb/audio_remap.c
index afc9c3209..d0374a0a0 100644
--- a/libhb/audio_remap.c
+++ b/libhb/audio_remap.c
@@ -246,19 +246,19 @@ fail:
 }
 
 void hb_audio_remap_set_channel_layout(hb_audio_remap_t *remap,
-                                       uint64_t channel_layout)
+                                       uint64_t channel_layout,
+                                       int channels)
 {
     if (remap != NULL)
     {
         int ii;
         remap->remap_needed = 0;
-        remap->nchannels    = av_get_channel_layout_nb_channels(channel_layout);
 
         // in some cases, remapping is not necessary and/or supported
-        if (remap->nchannels > HB_AUDIO_REMAP_MAX_CHANNELS)
+        if (channels > HB_AUDIO_REMAP_MAX_CHANNELS)
         {
             hb_log("hb_audio_remap_set_channel_layout: too many channels (%d)",
-                   remap->nchannels);
+                   channels);
             return;
         }
         if (remap->channel_map_in == remap->channel_map_out)
@@ -266,6 +266,14 @@ void hb_audio_remap_set_channel_layout(hb_audio_remap_t *remap,
             return;
         }
 
+        // sanitize the layout
+        channel_layout = hb_ff_layout_xlat(channel_layout, channels);
+        if (channel_layout == AV_CH_LAYOUT_STEREO_DOWNMIX)
+        {
+            channel_layout = AV_CH_LAYOUT_STEREO;
+        }
+        remap->nchannels = av_get_channel_layout_nb_channels(channel_layout);
+
         // build the table and check whether remapping is necessary
         hb_audio_remap_build_table(remap->channel_map_out,
                                    remap->channel_map_in, channel_layout,
diff --git a/libhb/audio_remap.h b/libhb/audio_remap.h
index 0bdd23bb7..3e4ab722b 100644
--- a/libhb/audio_remap.h
+++ b/libhb/audio_remap.h
@@ -73,7 +73,8 @@ hb_audio_remap_t* hb_audio_remap_init(enum AVSampleFormat sample_fmt,
  * Must be called at least once before remapping.
  */
 void              hb_audio_remap_set_channel_layout(hb_audio_remap_t *remap,
-                                                    uint64_t channel_layout);
+                                                    uint64_t channel_layout,
+                                                    int channels);
 
 /*
  * Free an hb_audio_remap_t.
diff --git a/libhb/deca52.c b/libhb/deca52.c
index ba7c38485..32da29ce1 100644
--- a/libhb/deca52.c
+++ b/libhb/deca52.c
@@ -380,7 +380,9 @@ static hb_buffer_t* Decode(hb_work_object_t *w)
         {
             pv->channel_layout = new_layout;
             pv->nchannels      = av_get_channel_layout_nb_channels(new_layout);
-            hb_audio_remap_set_channel_layout(pv->remap, pv->channel_layout);
+            hb_audio_remap_set_channel_layout(pv->remap,
+                                              pv->channel_layout,
+                                              pv->nchannels);
             hb_audio_resample_set_channel_layout(pv->resample,
                                                  pv->channel_layout,
                                                  pv->nchannels);
diff --git a/libhb/platform/macosx/encca_aac.c b/libhb/platform/macosx/encca_aac.c
index fe592e340..98e5ddfaf 100644
--- a/libhb/platform/macosx/encca_aac.c
+++ b/libhb/platform/macosx/encca_aac.c
@@ -296,7 +296,7 @@ int encCoreAudioInit(hb_work_object_t *w, hb_job_t *job, enum AAC_MODE mode)
         hb_error("encCoreAudioInit: hb_audio_remap_init() failed");
     }
     uint64_t layout = hb_ff_mixdown_xlat(audio->config.out.mixdown, NULL);
-    hb_audio_remap_set_channel_layout(pv->remap, layout);
+    hb_audio_remap_set_channel_layout(pv->remap, layout, pv->nchannels);
 
     // get maximum output size
     AudioConverterGetProperty(pv->converter,