audio_resample: fix mapping of mono to single channel (#593)

* audio_resample: fix mapping of mono to single channel

libav's mixer code can't map single channel layouts to other single
channel layouts. And we were asking it to map e.g. front left to front
center because out MONO mixdown == libav front center.  So when we
request MONO and the source is any single channel, change our output
layout to match the input.

fixes
https://forum.handbrake.fr/viewtopic.php?f=12&t=36016&sid=c5993fa7375792a940152c8adda19a54

* Incorporate Rodeo's suggestions

* fix grammar

1 files changed, 27 insertions, 0 deletions

diff --git a/libhb/audio_resample.c b/libhb/audio_resample.c
index edd84a589..35585dd66 100644
--- a/libhb/audio_resample.c
+++ b/libhb/audio_resample.c
@@ -75,6 +75,26 @@ fail:
     return NULL;
 }
 
+static int is_mono(uint64_t layout)
+{
+    int ii, channel_count;
+    int64_t mask;
+
+    if (layout == AV_CH_LAYOUT_NATIVE)
+    {
+        return 0;
+    }
+    for (ii = 0, channel_count = 0, mask = 1;
+         ii < 64 && channel_count < 2; ii++, mask <<= 1)
+    {
+        if (layout & mask)
+        {
+            channel_count++;
+        }
+    }
+    return channel_count == 1;
+}
+
 void hb_audio_resample_set_channel_layout(hb_audio_resample_t *resample,
                                           uint64_t channel_layout)
 {
@@ -85,6 +105,13 @@ void hb_audio_resample_set_channel_layout(hb_audio_resample_t *resample,
             // Dolby Surround is Stereo when it comes to remixing
             channel_layout = AV_CH_LAYOUT_STEREO;
         }
+        // avresample can't remap a single-channel layout to
+        // another single-channel layout
+        if (resample->out.channel_layout == AV_CH_LAYOUT_MONO &&
+            is_mono(channel_layout))
+        {
+            channel_layout = AV_CH_LAYOUT_MONO;
+        }
         resample->in.channel_layout = channel_layout;
     }
 }