qsv: fix AVC encoding hang when LA is enabled on Windows

4 files changed, 22 insertions, 11 deletions

diff --git a/libhb/enc_qsv.c b/libhb/enc_qsv.c
index fed7bb394..87c8dc0f6 100644
--- a/libhb/enc_qsv.c
+++ b/libhb/enc_qsv.c
@@ -2104,6 +2104,11 @@ static int qsv_enc_work(hb_work_private_t *pv,
 
             if (sts == MFX_ERR_MORE_DATA)
             {
+                if(!pv->is_sys_mem && surface)
+                {
+                    hb_qsv_release_surface_from_pool_by_surface_pointer(frames_ctx, surface);
+                }
+
                 if (qsv_atom != NULL)
                 {
                     hb_list_add(pv->delayed_processing, qsv_atom);
@@ -2132,6 +2137,11 @@ static int qsv_enc_work(hb_work_private_t *pv,
                 task->stage                    = new_stage;
                 pv->async_depth++;
 
+                if(!pv->is_sys_mem && surface)
+                {
+                    hb_qsv_release_surface_from_pool_by_surface_pointer(frames_ctx, surface);
+                }
+
                 if (qsv_atom != NULL)
                 {
                     hb_qsv_add_stagee(&qsv_atom, new_stage, HAVE_THREADS);
@@ -2176,13 +2186,6 @@ static int qsv_enc_work(hb_work_private_t *pv,
                 /* perform a sync operation to get the output bitstream */
                 hb_qsv_wait_on_sync(qsv_ctx, task->stage);
 
-                mfxFrameSurface1   *surface    = task->stage->in.p_surface;
-                HBQSVFramesContext *frames_ctx = task->stage->in.p_frames_ctx;
-                if(!pv->is_sys_mem && surface)
-                {
-                    hb_qsv_release_surface_from_pool_by_surface_pointer(frames_ctx, surface);
-                }
-
                 if (task->bs->DataLength > 0)
                 {
                     hb_qsv_list *pipe = hb_qsv_pipe_by_stage(qsv_ctx->pipes,
@@ -2270,7 +2273,7 @@ int encqsvWork(hb_work_object_t *w, hb_buffer_t **buf_in, hb_buffer_t **buf_out)
         else
         {
             // Create black buffer in the begining of the encoding, usually first 2 frames
-            hb_qsv_get_free_surface_from_pool_with_range(pv->job->qsv.ctx->hb_dec_qsv_frames_ctx, HB_QSV_POOL_SURFACE_SIZE - HB_QSV_POOL_ENCODER_SIZE, HB_QSV_POOL_SURFACE_SIZE, &mid, &surface);
+            hb_qsv_get_free_surface_from_pool_with_range(pv->job->qsv.ctx->hb_dec_qsv_frames_ctx, 0, HB_QSV_POOL_SURFACE_SIZE, &mid, &surface);
             frames_ctx = pv->job->qsv.ctx->hb_dec_qsv_frames_ctx;
         }
         hb_qsv_replace_surface_mid(frames_ctx, mid, surface);
diff --git a/libhb/handbrake/qsv_libav.h b/libhb/handbrake/qsv_libav.h
index 071325c9f..9ad40d338 100644
--- a/libhb/handbrake/qsv_libav.h
+++ b/libhb/handbrake/qsv_libav.h
@@ -212,7 +212,6 @@ typedef struct QSVFrame {
 
 #define HB_QSV_POOL_FFMPEG_SURFACE_SIZE (64)
 #define HB_QSV_POOL_SURFACE_SIZE (64)
-#define HB_QSV_POOL_ENCODER_SIZE (8)
 
 typedef struct HBQSVFramesContext {
     AVBufferRef *hw_frames_ctx;
diff --git a/libhb/qsv_common.c b/libhb/qsv_common.c
index 34b3fdcc0..2ca4f2874 100644
--- a/libhb/qsv_common.c
+++ b/libhb/qsv_common.c
@@ -3016,7 +3016,7 @@ hb_buffer_t* hb_qsv_copy_frame(hb_job_t *job, AVFrame *frame, int is_vpp)
     }
     else
     {
-        hb_qsv_get_free_surface_from_pool_with_range(hb_qsv_frames_ctx, 0, HB_QSV_POOL_SURFACE_SIZE - HB_QSV_POOL_ENCODER_SIZE, &mid, &output_surface);
+        hb_qsv_get_free_surface_from_pool_with_range(hb_qsv_frames_ctx, 0, HB_QSV_POOL_SURFACE_SIZE, &mid, &output_surface);
     }
 
     if (device_manager_handle_type == MFX_HANDLE_D3D9_DEVICE_MANAGER)
@@ -3228,7 +3228,9 @@ int hb_create_ffmpeg_pool(hb_job_t *job, int coded_width, int coded_height, enum
     frames_ctx->height            = FFALIGN(coded_height, 32);
     frames_ctx->format            = AV_PIX_FMT_QSV;
     frames_ctx->sw_format         = sw_pix_fmt;
-    frames_ctx->initial_pool_size = pool_size + extra_hw_frames;
+    frames_ctx->initial_pool_size = pool_size;
+    if (extra_hw_frames >= 0)
+        frames_ctx->initial_pool_size += extra_hw_frames;
     frames_hwctx->frame_type      = MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
 
     ret = av_hwframe_ctx_init(hw_frames_ctx);
diff --git a/libhb/sync.c b/libhb/sync.c
index c34a72a3f..9245bd370 100644
--- a/libhb/sync.c
+++ b/libhb/sync.c
@@ -2296,6 +2296,13 @@ static int syncVideoInit( hb_work_object_t * w, hb_job_t * job)
     pv->stream->in_queue        = hb_list_init();
     pv->stream->scr_delay_queue = hb_list_init();
     pv->stream->max_len         = SYNC_MAX_VIDEO_QUEUE_LEN;
+#if HB_PROJECT_FEATURE_QSV
+    // Fix of LA case allowing use of LA up to 40 in full encode path,
+    // as currently for such support we cannot allocate >64 slices per texture
+    // due to MSFT limitation, not impacting other cases
+    if (hb_qsv_full_path_is_enabled(job))
+        pv->stream->max_len     = SYNC_MIN_VIDEO_QUEUE_LEN;
+#endif
     pv->stream->min_len         = SYNC_MIN_VIDEO_QUEUE_LEN;
     if (pv->stream->in_queue == NULL) goto fail;
     pv->stream->delta_list      = hb_list_init();