glx: Properly handle cases where screen creation fails

This fixes a segfault exposed by a29d63ecf7 which occurs when swr is
used on an unsupported architecture.

v2: re-work to place logic in xmesa_init_display

Signed-off-by: Chuck Atkins <[email protected]>
Reviewed-by: Emil Velikov <[email protected]>
Cc: [email protected]
Cc: George Kyriazis <[email protected]>
Cc: Bruce Cherniak <[email protected]>

3 files changed, 33 insertions, 30 deletions

diff --git a/src/gallium/state_trackers/glx/xlib/glx_api.c b/src/gallium/state_trackers/glx/xlib/glx_api.c
index 1994e6823a8..6ec3a31d5fc 100644
--- a/src/gallium/state_trackers/glx/xlib/glx_api.c
+++ b/src/gallium/state_trackers/glx/xlib/glx_api.c
@@ -743,7 +743,10 @@ choose_visual( Display *dpy, int screen, const int *list, GLboolean fbConfig )
    int numAux = 0;
    GLint num_samples = 0;
 
-   xmesa_init( dpy );
+   if (xmesa_init( dpy ) != 0) {
+      _mesa_warning(NULL, "Failed to initialize display");
+      return NULL;
+   }
 
    parselist = list;
 
diff --git a/src/gallium/state_trackers/glx/xlib/xm_api.c b/src/gallium/state_trackers/glx/xlib/xm_api.c
index 934c0aba118..e8cbb618cea 100644
--- a/src/gallium/state_trackers/glx/xlib/xm_api.c
+++ b/src/gallium/state_trackers/glx/xlib/xm_api.c
@@ -142,8 +142,11 @@ xmesa_close_display(Display *display)
 {
    XMesaExtDisplayInfo *info, *prev;
 
+   /* These assertions are not valid since screen creation can fail and result
+    * in an empty list
    assert(MesaExtInfo.ndisplays > 0);
    assert(MesaExtInfo.head);
+   */
 
    _XLockMutex(_Xglobal_lock);
    /* first find display */
@@ -223,7 +226,30 @@ xmesa_init_display( Display *display )
       return NULL;
    }
    info->display = display;
+
    xmdpy = &info->mesaDisplay; /* to be filled out below */
+   xmdpy->display = display;
+   xmdpy->pipe = NULL;
+
+   xmdpy->smapi = CALLOC_STRUCT(st_manager);
+   if (!xmdpy->smapi) {
+      Xfree(info);
+      mtx_unlock(&init_mutex);
+      return NULL;
+   }
+
+   xmdpy->screen = driver.create_pipe_screen(display);
+   if (!xmdpy->screen) {
+      free(xmdpy->smapi);
+      Xfree(info);
+      mtx_unlock(&init_mutex);
+      return NULL;
+   }
+
+   /* At this point, both smapi and screen are known to be valid */
+   xmdpy->smapi->screen = xmdpy->screen;
+   xmdpy->smapi->get_param = xmesa_get_param;
+   (void) mtx_init(&xmdpy->mutex, mtx_plain);
 
    /* chain to the list of displays */
    _XLockMutex(_Xglobal_lock);
@@ -232,32 +258,6 @@ xmesa_init_display( Display *display )
    MesaExtInfo.ndisplays++;
    _XUnlockMutex(_Xglobal_lock);
 
-   /* now create the new XMesaDisplay info */
-   assert(display);
-
-   xmdpy->display = display;
-   xmdpy->screen = driver.create_pipe_screen(display);
-   xmdpy->smapi = CALLOC_STRUCT(st_manager);
-   xmdpy->pipe = NULL;
-   if (xmdpy->smapi) {
-      xmdpy->smapi->screen = xmdpy->screen;
-      xmdpy->smapi->get_param = xmesa_get_param;
-   }
-
-   if (xmdpy->screen && xmdpy->smapi) {
-      (void) mtx_init(&xmdpy->mutex, mtx_plain);
-   }
-   else {
-      if (xmdpy->screen) {
-         xmdpy->screen->destroy(xmdpy->screen);
-         xmdpy->screen = NULL;
-      }
-      free(xmdpy->smapi);
-      xmdpy->smapi = NULL;
-
-      xmdpy->display = NULL;
-   }
-
    mtx_unlock(&init_mutex);
 
    return xmdpy;
@@ -935,10 +935,10 @@ xmesa_get_name(void)
 /**
  * Do per-display initializations.
  */
-void
+int
 xmesa_init( Display *display )
 {
-   xmesa_init_display(display);
+   return xmesa_init_display(display) ? 0 : 1;
 }
 
 
diff --git a/src/gallium/state_trackers/glx/xlib/xm_api.h b/src/gallium/state_trackers/glx/xlib/xm_api.h
index 06bf839c96e..e420679313e 100644
--- a/src/gallium/state_trackers/glx/xlib/xm_api.h
+++ b/src/gallium/state_trackers/glx/xlib/xm_api.h
@@ -359,7 +359,7 @@ struct xmesa_buffer {
 extern const char *
 xmesa_get_name(void);
 
-extern void
+extern int
 xmesa_init(Display *dpy);
 
 extern XMesaBuffer