LinGui: make QSV encoding actually work

Assuming you built HandBrake with 'configure --enable-qsv' and you have
built and installed Intel MediaSDK in a directory that is in your LD search
path, the QSV encoders now work.  HW decode is not supported.

5 files changed, 219 insertions, 7 deletions

diff --git a/libhb/enc_qsv.c b/libhb/enc_qsv.c
index 4c2aa8e49..53b87eb3b 100644
--- a/libhb/enc_qsv.c
+++ b/libhb/enc_qsv.c
@@ -75,6 +75,7 @@ struct hb_work_private_s
     hb_qsv_param_t       param;
     hb_qsv_space         enc_space;
     hb_qsv_info_t      * qsv_info;
+    hb_display_t       * display;
 
     hb_chapter_queue_t * chapter_queue;
 
@@ -1084,6 +1085,18 @@ int encqsvInit(hb_work_object_t *w, hb_job_t *job)
         return -1;
     }
 
+    if (pv->qsv_info->implementation & MFX_IMPL_HARDWARE_ANY)
+    {
+        // On linux, the handle to the VA display must be set.
+        // This code is essentiall a NOP other platforms.
+        pv->display = hb_qsv_display_init();
+        if (pv->display != NULL)
+        {
+            MFXVideoCORE_SetHandle(session, pv->display->mfxType,
+                                   (mfxHDL)pv->display->handle);
+        }
+    }
+
     /* Query the API version for hb_qsv_load_plugins */
     err = MFXQueryVersion(session, &version);
     if (err != MFX_ERR_NONE)
@@ -1430,6 +1443,8 @@ void encqsvClose(hb_work_object_t *w)
             /* QSV context cleanup and MFXClose */
             hb_qsv_context_clean(qsv_ctx);
 
+            hb_display_close(&pv->display);
+
             if (qsv_enc_space != NULL)
             {
                 if (qsv_enc_space->is_init_done)
diff --git a/libhb/ports.c b/libhb/ports.c
index 6773f2905..c9d2dcd11 100644
--- a/libhb/ports.c
+++ b/libhb/ports.c
@@ -70,6 +70,7 @@
 #include <linux/cdrom.h>
 #include <fcntl.h>
 #include <sys/ioctl.h>
+#include <libdrm/drm.h>
 #elif defined( SYS_OPENBSD )
 #include <sys/dvdio.h>
 #include <fcntl.h>
@@ -1505,3 +1506,150 @@ char * hb_strndup(const char * src, size_t len)
     return strndup(src, len);
 #endif
 }
+
+#ifdef USE_QSV
+#ifdef SYS_LINUX
+
+#define MAX_NODES             16
+#define DRI_RENDER_NODE_START 128
+#define DRI_RENDER_NODE_LAST  (DRI_RENDER_NODE_START + MAX_NODES - 1)
+#define DRI_CARD_NODE_START   0
+#define DRI_CARD_NODE_LAST    (DRI_CARD_NODE_START + MAX_NODES - 1)
+
+const char* DRI_PATH = "/dev/dri/";
+const char* DRI_NODE_RENDER = "renderD";
+const char* DRI_NODE_CARD = "card";
+
+static int try_adapter(const char * name, const char * dir,
+                const char * prefix, int node_start, int node_last)
+{
+    int             node;
+    int             len        = strlen(name);
+    char          * driverName = malloc(len + 1);
+    drm_version_t   version = {};
+
+    version.name_len = len + 1;
+    version.name     = driverName;
+    for (node = node_start; node <= node_last; node++)
+    {
+        char * adapter = hb_strdup_printf("%s%s%d", dir, prefix, node);
+        int    fd      = open(adapter, O_RDWR);
+
+        free(adapter);
+        if (fd < 0)
+        {
+            continue;
+        }
+
+        if (!ioctl(fd, DRM_IOCTL_VERSION, &version) &&
+            version.name_len == len && !strncmp(driverName, name, len))
+        {
+            free(driverName);
+            return fd;
+        }
+        close(fd);
+    }
+
+    free(driverName);
+    return -1;
+}
+
+static int open_adapter(const char * name)
+{
+    int fd = try_adapter(name, DRI_PATH, DRI_NODE_RENDER,
+                         DRI_RENDER_NODE_START, DRI_RENDER_NODE_LAST);
+    if (fd < 0)
+    {
+        fd = try_adapter(name, DRI_PATH, DRI_NODE_CARD,
+                         DRI_CARD_NODE_START, DRI_CARD_NODE_LAST);
+    }
+    return fd;
+}
+
+hb_display_t * hb_display_init(const char * driver_name,
+                               const char * interface_name)
+{
+    hb_display_t * hbDisplay = calloc(sizeof(hb_display_t), 1);
+
+    hbDisplay->vaDisplay = NULL;
+    hbDisplay->vaFd      = open_adapter(driver_name);
+    if (hbDisplay->vaFd < 0)
+    {
+        hb_deep_log( 3, "hb_va_display_init: no display found" );
+        free(hbDisplay);
+        return NULL;
+    }
+
+    setenv("LIBVA_DRIVER_NAME", interface_name, 1);
+    hbDisplay->vaDisplay = vaGetDisplayDRM(hbDisplay->vaFd);
+    if (hbDisplay->vaDisplay == NULL)
+    {
+        close(hbDisplay->vaFd);
+        free(hbDisplay);
+        return NULL;
+    }
+
+    int major = 0, minor = 0;
+    VAStatus vaRes = vaInitialize(hbDisplay->vaDisplay, &major, &minor);
+    if (vaRes != VA_STATUS_SUCCESS)
+    {
+        vaTerminate(hbDisplay->vaDisplay);
+        close(hbDisplay->vaFd);
+        free(hbDisplay);
+        return NULL;
+    }
+    hbDisplay->handle = hbDisplay->vaDisplay;
+    hbDisplay->mfxType = MFX_HANDLE_VA_DISPLAY;
+
+    return hbDisplay;
+}
+
+void hb_display_close(hb_display_t ** _d)
+{
+    hb_display_t * hbDisplay = *_d;
+
+    if (hbDisplay == NULL)
+    {
+        return;
+    }
+    if (hbDisplay->vaDisplay)
+    {
+        vaTerminate(hbDisplay->vaDisplay);
+    }
+    if (hbDisplay->vaFd >= 0)
+    {
+        close(hbDisplay->vaFd);
+    }
+    free(hbDisplay);
+
+    *_d = NULL;
+}
+
+#else // !SYS_LINUX
+
+hb_display_t * hb_display_init(const char * driver_name,
+                               const char * interface_name)
+{
+    return NULL;
+}
+
+void hb_display_close(hb_display_t ** _d)
+{
+    (void)_d;
+}
+
+#endif // SYS_LINUX
+#else // !USE_QSV
+
+hb_display_t * hb_display_init(const char * driver_name,
+                               const char * interface_name)
+{
+    return NULL;
+}
+
+void hb_display_close(hb_display_t ** _d)
+{
+    (void)_d;
+}
+
+#endif // USE_QSV
diff --git a/libhb/ports.h b/libhb/ports.h
index 6a6c3c755..02e4ebd5b 100644
--- a/libhb/ports.h
+++ b/libhb/ports.h
@@ -24,6 +24,36 @@
 #define IS_DIR_SEP(c) (c == '/')
 #endif
 
+#ifdef USE_QSV
+#include "mfx/mfxstructures.h"
+#ifdef SYS_LINUX
+#include <va/va_drm.h>
+#endif
+#endif
+
+/************************************************************************
+ * HW accel display
+ ***********************************************************************/
+#ifdef SYS_LINUX
+extern const char* DRM_INTEL_DRIVER_NAME;
+#endif // SYS_LINUX
+
+typedef struct
+{
+    void          * handle;
+#ifdef USE_QSV
+    mfxHandleType   mfxType;
+
+#ifdef SYS_LINUX
+    int             vaFd;
+    VADisplay       vaDisplay;
+#endif // SYS_LINUX
+#endif
+} hb_display_t;
+
+hb_display_t * hb_display_init(const char * driver_name,
+                               const char * interface_name);
+void           hb_display_close(hb_display_t ** _d);
 
 /************************************************************************
  * CPU info utilities
diff --git a/libhb/qsv_common.c b/libhb/qsv_common.c
index 7376be4b9..f3f3082a7 100644
--- a/libhb/qsv_common.c
+++ b/libhb/qsv_common.c
@@ -316,13 +316,15 @@ static int query_capabilities(mfxSession session, mfxVersion version, hb_qsv_inf
         }
         else
         {
+            mfxStatus mfxRes;
             init_video_param(&inputParam);
             inputParam.mfx.CodecId = info->codec_id;
 
             memset(&videoParam, 0, sizeof(mfxVideoParam));
             videoParam.mfx.CodecId = inputParam.mfx.CodecId;
 
-            if (MFXVideoENCODE_Query(session, &inputParam, &videoParam) >= MFX_ERR_NONE &&
+            mfxRes = MFXVideoENCODE_Query(session, &inputParam, &videoParam);
+            if (mfxRes >= MFX_ERR_NONE &&
                 videoParam.mfx.CodecId == info->codec_id)
             {
                 /*
@@ -636,6 +638,14 @@ static int query_capabilities(mfxSession session, mfxVersion version, hb_qsv_inf
     return 0;
 }
 
+const char* DRM_INTEL_DRIVER_NAME = "i915";
+const char* VA_INTEL_DRIVER_NAME = "iHD";
+
+hb_display_t * hb_qsv_display_init(void)
+{
+    return hb_display_init(DRM_INTEL_DRIVER_NAME, VA_INTEL_DRIVER_NAME);
+}
+
 int hb_qsv_info_init()
 {
     static int init_done = 0;
@@ -680,6 +690,15 @@ int hb_qsv_info_init()
     do{
         if (MFXInit(MFX_IMPL_HARDWARE_ANY | hw_preference, &version, &session) == MFX_ERR_NONE)
         {
+            // On linux, the handle to the VA display must be set.
+            // This code is essentiall a NOP other platforms.
+            hb_display_t * display = hb_qsv_display_init();
+
+            if (display != NULL)
+            {
+                MFXVideoCORE_SetHandle(session, display->mfxType,
+                                       (mfxHDL)display->handle);
+            }
             // Media SDK hardware found, but check that our minimum is supported
             //
             // Note: this-party hardware (QSV_G0) is unsupported for the time being
@@ -697,6 +716,7 @@ int hb_qsv_info_init()
                 // available, we can set the preferred implementation
                 hb_qsv_impl_set_preferred("hardware");
             }
+            hb_display_close(&display);
             MFXClose(session);
             hw_preference = 0;
         }
@@ -897,19 +917,17 @@ hb_list_t* hb_qsv_load_plugins(hb_qsv_info_t *info, mfxSession session, mfxVersi
             if (HB_CHECK_MFX_VERSION(version, 1, 15) &&
                 qsv_implementation_is_hardware(info->implementation))
             {
-                if (MFXVideoUSER_Load(session, &MFX_PLUGINID_HEVCE_HW, 0) < MFX_ERR_NONE)
+                if (MFXVideoUSER_Load(session, &MFX_PLUGINID_HEVCE_HW, 0) == MFX_ERR_NONE)
                 {
-                    goto fail;
+                    hb_list_add(mfxPluginList, (void*)&MFX_PLUGINID_HEVCE_HW);
                 }
-                hb_list_add(mfxPluginList, (void*)&MFX_PLUGINID_HEVCE_HW);
             }
             else if (HB_CHECK_MFX_VERSION(version, 1, 15))
             {
-                if (MFXVideoUSER_Load(session, &MFX_PLUGINID_HEVCE_SW, 0) < MFX_ERR_NONE)
+                if (MFXVideoUSER_Load(session, &MFX_PLUGINID_HEVCE_SW, 0) == MFX_ERR_NONE)
                 {
-                    goto fail;
+                    hb_list_add(mfxPluginList, (void*)&MFX_PLUGINID_HEVCE_SW);
                 }
-                hb_list_add(mfxPluginList, (void*)&MFX_PLUGINID_HEVCE_SW);
             }
         }
     }
diff --git a/libhb/qsv_common.h b/libhb/qsv_common.h
index 04bd0b40e..4b690a661 100644
--- a/libhb/qsv_common.h
+++ b/libhb/qsv_common.h
@@ -68,6 +68,7 @@ typedef struct hb_qsv_info_s
 } hb_qsv_info_t;
 
 /* Intel Quick Sync Video utilities */
+hb_display_t * hb_qsv_display_init(void);
 int            hb_qsv_video_encoder_is_enabled(int encoder);
 int            hb_qsv_audio_encoder_is_enabled(int encoder);
 int            hb_qsv_info_init();