libav: fix decoding of mpeg4 in transport streams

There was a chicken & egg problem in the initialization sequence.  mpeg4 only
worked in non-transport streams because we use libav for demuxing these.
The initializion of libav demux causes an additional initialization of the
codec which hides the problem.


git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@6272 b64f7644-9d1e-0410-96f1-a4d463321fa5

1 files changed, 41 insertions, 0 deletions

diff --git a/contrib/ffmpeg/A05-mpeg4-initialization.patch b/contrib/ffmpeg/A05-mpeg4-initialization.patch
new file mode 100644
index 000000000..3045c8d28
--- /dev/null
+++ b/contrib/ffmpeg/A05-mpeg4-initialization.patch
@@ -0,0 +1,41 @@
+diff --git a/libavcodec/h263dec.c b/libavcodec/h263dec.c
+index f6c79af..70522ee 100644
+--- a/libavcodec/h263dec.c
++++ b/libavcodec/h263dec.c
+@@ -414,17 +414,7 @@ int ff_h263_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
+ 
+     if (!s->context_initialized)
+         // we need the idct permutaton for reading a custom matrix
+-        if ((ret = ff_MPV_common_init(s)) < 0)
+-            return ret;
+-
+-    /* We need to set current_picture_ptr before reading the header,
+-     * otherwise we cannot store anyting in there */
+-    if (s->current_picture_ptr == NULL || s->current_picture_ptr->f.data[0]) {
+-        int i = ff_find_unused_picture(s, 0);
+-        if (i < 0)
+-            return i;
+-        s->current_picture_ptr = &s->picture[i];
+-    }
++        ff_dsputil_init(&s->dsp, avctx);
+ 
+     /* let's go :-) */
+     if (CONFIG_WMV2_DECODER && s->msmpeg4_version == 5) {
+@@ -459,6 +449,17 @@ int ff_h263_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
+         return ret;
+     }
+ 
++    if (!s->context_initialized)
++        if ((ret = ff_MPV_common_init(s)) < 0)
++            return ret;
++
++    if (s->current_picture_ptr == NULL || s->current_picture_ptr->f.data[0]) {
++        int i = ff_find_unused_picture(s, 0);
++        if (i < 0)
++            return i;
++        s->current_picture_ptr = &s->picture[i];
++    }
++
+     avctx->has_b_frames = !s->low_delay;
+ 
+ #define SET_QPEL_FUNC(postfix1, postfix2)                           \