libav: fix latm aac decoder hang

Don't return EAGAIN from decoders.

When waiting for codec parameters, the decoder returned EAGAIN which was
meant to be interpreted as "send more data". But the new libav decoder
API changed the definition of EAGAIN to mean "send the same packet
again".  So hang.

1 files changed, 71 insertions, 0 deletions

diff --git a/contrib/ffmpeg/A16-decoder-hang-eagain.patch b/contrib/ffmpeg/A16-decoder-hang-eagain.patch
new file mode 100644
index 000000000..5b59cae93
--- /dev/null
+++ b/contrib/ffmpeg/A16-decoder-hang-eagain.patch
@@ -0,0 +1,71 @@
+diff --git a/libavcodec/aacdec.c b/libavcodec/aacdec.c
+index 6a06062..0e629ce 100644
+--- a/libavcodec/aacdec.c
++++ b/libavcodec/aacdec.c
+@@ -3229,7 +3229,7 @@ static int read_audio_mux_element(struct LATMContext *latmctx,
+     } else if (!latmctx->aac_ctx.avctx->extradata) {
+         av_log(latmctx->aac_ctx.avctx, AV_LOG_DEBUG,
+                "no decoder config found\n");
+-        return AVERROR(EAGAIN);
++        return 1;
+     }
+     if (latmctx->audio_mux_version_A == 0) {
+         int mux_slot_length_bytes = read_payload_length_info(latmctx, gb);
+@@ -3266,8 +3266,8 @@ static int latm_decode_frame(AVCodecContext *avctx, void *out,
+     if (muxlength > avpkt->size)
+         return AVERROR_INVALIDDATA;
+ 
+-    if ((err = read_audio_mux_element(latmctx, &gb)) < 0)
+-        return err;
++    if ((err = read_audio_mux_element(latmctx, &gb)))
++        return (err < 0) ? err : avpkt->size;
+ 
+     if (!latmctx->initialized) {
+         if (!avctx->extradata) {
+diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
+index 1ff27a1..3c1528c 100644
+--- a/libavcodec/nvenc.c
++++ b/libavcodec/nvenc.c
+@@ -113,12 +113,14 @@ static const struct {
+     { NV_ENC_ERR_OUT_OF_MEMORY,            AVERROR(ENOMEM),  "out of memory"            },
+     { NV_ENC_ERR_ENCODER_NOT_INITIALIZED,  AVERROR(EINVAL),  "encoder not initialized"  },
+     { NV_ENC_ERR_UNSUPPORTED_PARAM,        AVERROR(ENOSYS),  "unsupported param"        },
+-    { NV_ENC_ERR_LOCK_BUSY,                AVERROR(EAGAIN),  "lock busy"                },
++    { NV_ENC_ERR_LOCK_BUSY,                AVERROR(EBUSY),   "lock busy"                },
+     { NV_ENC_ERR_NOT_ENOUGH_BUFFER,        AVERROR(ENOBUFS), "not enough buffer"        },
+     { NV_ENC_ERR_INVALID_VERSION,          AVERROR(EINVAL),  "invalid version"          },
+     { NV_ENC_ERR_MAP_FAILED,               AVERROR(EIO),     "map failed"               },
+-    { NV_ENC_ERR_NEED_MORE_INPUT,          AVERROR(EAGAIN),  "need more input"          },
+-    { NV_ENC_ERR_ENCODER_BUSY,             AVERROR(EAGAIN),  "encoder busy"             },
++    /* this is error should always be treated specially, so this "mapping"
++     * is for completeness only */
++    { NV_ENC_ERR_NEED_MORE_INPUT,          AVERROR_UNKNOWN,  "need more input"          },
++    { NV_ENC_ERR_ENCODER_BUSY,             AVERROR(EBUSY),   "encoder busy"             },
+     { NV_ENC_ERR_EVENT_NOT_REGISTERD,      AVERROR(EBADF),   "event not registered"     },
+     { NV_ENC_ERR_GENERIC,                  AVERROR_UNKNOWN,  "generic error"            },
+     { NV_ENC_ERR_INCOMPATIBLE_CLIENT_KEY,  AVERROR(EINVAL),  "incompatible client key"  },
+diff --git a/libavcodec/qsv.c b/libavcodec/qsv.c
+index 9119586..45a353a 100644
+--- a/libavcodec/qsv.c
++++ b/libavcodec/qsv.c
+@@ -69,15 +69,17 @@ static const struct {
+     { MFX_ERR_LOCK_MEMORY,              AVERROR(EIO),    "failed to lock the memory block"      },
+     { MFX_ERR_NOT_INITIALIZED,          AVERROR_BUG,     "not initialized"                      },
+     { MFX_ERR_NOT_FOUND,                AVERROR(ENOSYS), "specified object was not found"       },
+-    { MFX_ERR_MORE_DATA,                AVERROR(EAGAIN), "expect more data at input"            },
+-    { MFX_ERR_MORE_SURFACE,             AVERROR(EAGAIN), "expect more surface at output"        },
++    /* the following 3 errors should always be handled explicitly, so those "mappings"
++     * are for completeness only */
++    { MFX_ERR_MORE_DATA,                AVERROR_UNKNOWN, "expect more data at input"            },
++    { MFX_ERR_MORE_SURFACE,             AVERROR_UNKNOWN, "expect more surface at output"        },
++    { MFX_ERR_MORE_BITSTREAM,           AVERROR_UNKNOWN, "expect more bitstream at output"      },
+     { MFX_ERR_ABORTED,                  AVERROR_UNKNOWN, "operation aborted"                    },
+     { MFX_ERR_DEVICE_LOST,              AVERROR(EIO),    "device lost"                          },
+     { MFX_ERR_INCOMPATIBLE_VIDEO_PARAM, AVERROR(EINVAL), "incompatible video parameters"        },
+     { MFX_ERR_INVALID_VIDEO_PARAM,      AVERROR(EINVAL), "invalid video parameters"             },
+     { MFX_ERR_UNDEFINED_BEHAVIOR,       AVERROR_BUG,     "undefined behavior"                   },
+     { MFX_ERR_DEVICE_FAILED,            AVERROR(EIO),    "device failed"                        },
+-    { MFX_ERR_MORE_BITSTREAM,           AVERROR(EAGAIN), "expect more bitstream at output"      },
+     { MFX_ERR_INCOMPATIBLE_AUDIO_PARAM, AVERROR(EINVAL), "incompatible audio parameters"        },
+     { MFX_ERR_INVALID_AUDIO_PARAM,      AVERROR(EINVAL), "invalid audio parameters"             },
+