st/vdpau: error handling

handle the cases when vl_compositor_set_csc_matrix(),
vl_compositor_init_state() and vl_compositor_init() fail

Signed-off-by: Nayan Deshmukh <[email protected]>
Reviewed-by: Christian König <[email protected]>

3 files changed, 50 insertions, 15 deletions

diff --git a/src/gallium/state_trackers/vdpau/device.c b/src/gallium/state_trackers/vdpau/device.c
index 81b75827f34..8bae0647720 100644
--- a/src/gallium/state_trackers/vdpau/device.c
+++ b/src/gallium/state_trackers/vdpau/device.c
@@ -128,13 +128,19 @@ vdp_imp_device_create_x11(Display *display, int screen, VdpDevice *device,
       goto no_handle;
    }
 
-   vl_compositor_init(&dev->compositor, dev->context);
+   if (!vl_compositor_init(&dev->compositor, dev->context)) {
+       ret = VDP_STATUS_ERROR;
+       goto no_compositor;
+   }
+
    pipe_mutex_init(dev->mutex);
 
    *get_proc_address = &vlVdpGetProcAddress;
 
    return VDP_STATUS_OK;
 
+no_compositor:
+   vlRemoveDataHTAB(*device);
 no_handle:
    pipe_sampler_view_reference(&dev->dummy_sv, NULL);
 no_resource:
diff --git a/src/gallium/state_trackers/vdpau/mixer.c b/src/gallium/state_trackers/vdpau/mixer.c
index aca43c1e26e..10141748968 100644
--- a/src/gallium/state_trackers/vdpau/mixer.c
+++ b/src/gallium/state_trackers/vdpau/mixer.c
@@ -65,11 +65,18 @@ vlVdpVideoMixerCreate(VdpDevice device,
 
    pipe_mutex_lock(dev->mutex);
 
-   vl_compositor_init_state(&vmixer->cstate, dev->context);
+   if (!vl_compositor_init_state(&vmixer->cstate, dev->context)) {
+      ret = VDP_STATUS_ERROR;
+      goto no_compositor_state;
+   }
 
    vl_csc_get_matrix(VL_CSC_COLOR_STANDARD_BT_601, NULL, true, &vmixer->csc);
-   if (!debug_get_bool_option("G3DVL_NO_CSC", FALSE))
-      vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc, 1.0f, 0.0f);
+   if (!debug_get_bool_option("G3DVL_NO_CSC", FALSE)) {
+      if (!vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc, 1.0f, 0.0f)) {
+         ret = VDP_STATUS_ERROR;
+         goto err_csc_matrix;
+      }
+   }
 
    *mixer = vlAddDataHTAB(vmixer);
    if (*mixer == 0) {
@@ -163,7 +170,9 @@ no_params:
    vlRemoveDataHTAB(*mixer);
 
 no_handle:
+err_csc_matrix:
    vl_compositor_cleanup_state(&vmixer->cstate);
+no_compositor_state:
    pipe_mutex_unlock(dev->mutex);
    DeviceReference(&vmixer->device, NULL);
    FREE(vmixer);
@@ -690,8 +699,11 @@ vlVdpVideoMixerSetFeatureEnables(VdpVideoMixer mixer,
       case VDP_VIDEO_MIXER_FEATURE_LUMA_KEY:
          vmixer->luma_key.enabled = feature_enables[i];
          if (!debug_get_bool_option("G3DVL_NO_CSC", FALSE))
-            vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc,
-                                         vmixer->luma_key.luma_min, vmixer->luma_key.luma_max);
+            if (!vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc,
+                        vmixer->luma_key.luma_min, vmixer->luma_key.luma_max)) {
+               pipe_mutex_unlock(vmixer->device->mutex);
+               return VDP_STATUS_ERROR;
+            }
          break;
 
       case VDP_VIDEO_MIXER_FEATURE_HIGH_QUALITY_SCALING_L1:
@@ -810,8 +822,11 @@ vlVdpVideoMixerSetAttributeValues(VdpVideoMixer mixer,
          else
             memcpy(vmixer->csc, vdp_csc, sizeof(vl_csc_matrix));
          if (!debug_get_bool_option("G3DVL_NO_CSC", FALSE))
-            vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc,
-                                         vmixer->luma_key.luma_min, vmixer->luma_key.luma_max);
+            if (!vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc,
+                                         vmixer->luma_key.luma_min, vmixer->luma_key.luma_max)) {
+               ret = VDP_STATUS_ERROR;
+               goto fail;
+            }
          break;
 
       case VDP_VIDEO_MIXER_ATTRIBUTE_NOISE_REDUCTION_LEVEL:
@@ -834,8 +849,11 @@ vlVdpVideoMixerSetAttributeValues(VdpVideoMixer mixer,
          }
          vmixer->luma_key.luma_min = val;
          if (!debug_get_bool_option("G3DVL_NO_CSC", FALSE))
-            vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc,
-                                         vmixer->luma_key.luma_min, vmixer->luma_key.luma_max);
+            if (!vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc,
+                        vmixer->luma_key.luma_min, vmixer->luma_key.luma_max)) {
+               ret = VDP_STATUS_ERROR;
+               goto fail;
+            }
          break;
 
       case VDP_VIDEO_MIXER_ATTRIBUTE_LUMA_KEY_MAX_LUMA:
@@ -846,8 +864,11 @@ vlVdpVideoMixerSetAttributeValues(VdpVideoMixer mixer,
          }
          vmixer->luma_key.luma_max = val;
          if (!debug_get_bool_option("G3DVL_NO_CSC", FALSE))
-            vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc,
-                                         vmixer->luma_key.luma_min, vmixer->luma_key.luma_max);
+            if (!vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc,
+                        vmixer->luma_key.luma_min, vmixer->luma_key.luma_max)) {
+               ret = VDP_STATUS_ERROR;
+               goto fail;
+            }
          break;
 
       case VDP_VIDEO_MIXER_ATTRIBUTE_SHARPNESS_LEVEL:
diff --git a/src/gallium/state_trackers/vdpau/output.c b/src/gallium/state_trackers/vdpau/output.c
index 8c29a3ff4e5..64574b23762 100644
--- a/src/gallium/state_trackers/vdpau/output.c
+++ b/src/gallium/state_trackers/vdpau/output.c
@@ -111,7 +111,9 @@ vlVdpOutputSurfaceCreate(VdpDevice device,
 
    pipe_resource_reference(&res, NULL);
 
-   vl_compositor_init_state(&vlsurface->cstate, pipe);
+   if (!vl_compositor_init_state(&vlsurface->cstate, pipe))
+      goto err_resource;
+
    vl_compositor_reset_dirty_area(&vlsurface->dirty_area);
    pipe_mutex_unlock(dev->mutex);
 
@@ -492,9 +494,11 @@ vlVdpOutputSurfacePutBitsYCbCr(VdpOutputSurface surface,
    if (!csc_matrix) {
       vl_csc_matrix csc;
       vl_csc_get_matrix(VL_CSC_COLOR_STANDARD_BT_601, NULL, 1, &csc);
-      vl_compositor_set_csc_matrix(cstate, (const vl_csc_matrix*)&csc, 1.0f, 0.0f);
+      if (!vl_compositor_set_csc_matrix(cstate, (const vl_csc_matrix*)&csc, 1.0f, 0.0f))
+         goto err_csc_matrix;
    } else {
-      vl_compositor_set_csc_matrix(cstate, csc_matrix, 1.0f, 0.0f);
+      if (!vl_compositor_set_csc_matrix(cstate, csc_matrix, 1.0f, 0.0f))
+         goto err_csc_matrix;
    }
 
    vl_compositor_clear_layers(cstate);
@@ -506,6 +510,10 @@ vlVdpOutputSurfacePutBitsYCbCr(VdpOutputSurface surface,
    pipe_mutex_unlock(vlsurface->device->mutex);
 
    return VDP_STATUS_OK;
+err_csc_matrix:
+   vbuffer->destroy(vbuffer);
+   pipe_mutex_unlock(vlsurface->device->mutex);
+   return VDP_STATUS_ERROR;
 }
 
 static unsigned