MacGui: Add flexible dylib path manipulation so that we can handle external dylibs more easily

At startup, add any extra dylib paths to DYLD_FALLBACK_LIBRARY_PATH.  This is the last path
searched by the system for dylibs and we add our paths to the end of it's list, so this
will never override any system libs or paths the user has explicitly set.

Since applications read the environment once at startup, these changes don't take effect
until we restart with execv.  In order to avoid a possible exec bomb, we add a parameter to
the argv list to prevent any further exec's.

Note that this causes a minor glitch when running under gdb. The execv triggers a trap.  You
just have to 'continue'.


git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@3351 b64f7644-9d1e-0410-96f1-a4d463321fa5

2 files changed, 122 insertions, 124 deletions

diff --git a/macosx/Controller.m b/macosx/Controller.m
index 6526d1598..5a85cd2c0 100644
--- a/macosx/Controller.m
+++ b/macosx/Controller.m
@@ -4,6 +4,7 @@
    Homepage: <http://handbrake.fr/>.
    It may be used under the terms of the GNU General Public License. */
 
+#include <dlfcn.h>
 #import "Controller.h"
 #import "HBOutputPanelController.h"
 #import "HBPreferencesController.h"
@@ -1614,31 +1615,20 @@ static NSString *        ChooseSourceIdentifier             = @"Choose Source It
     
     if( [detector isVideoDVD] )
     {
-        int hb_arch;
-#if defined( __LP64__ )
-        /* we are 64 bit */
-        hb_arch = 64;
-#else
-        /* we are 32 bit */
-        hb_arch = 32;
-#endif 
-        
-        
         // The chosen path was actually on a DVD, so use the raw block
         // device path instead.
         path = [detector devicePath];
         [self writeToActivityLog: "trying to open a physical dvd at: %s", [scanPath UTF8String]];
         
         /* lets check for vlc here to make sure we have a dylib available to use for decrypting */
-        NSString *vlcPath = @"/Applications/VLC.app/Contents/MacOS/lib/libdvdcss.2.dylib";
-        NSFileManager * fileManager = [NSFileManager defaultManager];
-	    if ([fileManager fileExistsAtPath:vlcPath] == 0) 
-	    {
-            /*vlc not found in /Applications so we set the bool to cancel scanning to 1 */
+        void *dvdcss = dlopen("libdvdcss.2.dylib", RTLD_LAZY);
+        if (dvdcss == NULL) 
+        {
+            /*compatible vlc not found, so we set the bool to cancel scanning to 1 */
             cancelScanDecrypt = 1;
             [self writeToActivityLog: "VLC app not found for decrypting physical dvd"];
             int status;
-            status = NSRunAlertPanel(@"HandBrake could not find VLC or your VLC is out of date.",@"Please download and install VLC media player in your /Applications folder if you wish to read encrypted DVDs.", @"Get VLC", @"Cancel Scan", @"Attempt Scan Anyway");
+            status = NSRunAlertPanel(@"HandBrake could not find VLC or your VLC is incompatible (Note: 32 bit vlc is not compatible with 64 bit HandBrake and vice-versa).",@"Please download and install VLC media player if you wish to read encrypted DVDs.", @"Get VLC", @"Cancel Scan", @"Attempt Scan Anyway");
             [NSApp requestUserAttention:NSCriticalRequest];
             
             if (status == NSAlertDefaultReturn)
@@ -1664,111 +1654,8 @@ static NSString *        ChooseSourceIdentifier             = @"Choose Source It
             /* VLC was found in /Applications so all is well, we can carry on using vlc's libdvdcss.dylib for decrypting if needed */
             [self writeToActivityLog: "VLC app found for decrypting physical dvd"];
             vlcFound = 1;
+            dlclose(dvdcss);
         }
-        /* test for architecture of the vlc app */
-        NSArray *vlc_architecturesArray = [[NSBundle bundleWithPath:@"/Applications/VLC.app"] executableArchitectures];
-        BOOL vlcIntel32bit = NO;
-        BOOL vlcIntel64bit = NO;
-        BOOL vlcPPC32bit = NO;
-        BOOL vlcPPC64bit = NO;
-        /* check the available architectures for vlc and note accordingly */
-        NSEnumerator *enumerator = [vlc_architecturesArray objectEnumerator];
-        id tempObject;
-        while (tempObject = [enumerator nextObject])
-        {
-            
-            if ([tempObject intValue] == NSBundleExecutableArchitectureI386)
-            {
-                vlcIntel32bit = YES;   
-            }
-            if ([tempObject intValue] == NSBundleExecutableArchitectureX86_64)
-            {
-                vlcIntel64bit = YES;   
-            }
-            if ([tempObject intValue] == NSBundleExecutableArchitecturePPC)
-            {
-                vlcPPC32bit = YES;   
-            }
-            if ([tempObject intValue] == NSBundleExecutableArchitecturePPC64)
-            {
-                vlcPPC64bit = YES;   
-            }
-            
-        }
-        /* Write vlc architecture findings to activity window */
-        if (vlcIntel32bit)
-        {
-            [self writeToActivityLog: " 32-Bit VLC app found for decrypting physical dvd"];
-        }
-        if (vlcIntel64bit)
-        {
-            [self writeToActivityLog: " 64-Bit VLC app found for decrypting physical dvd"];
-        }
-        
-        
-        
-        if (vlcFound && hb_arch == 64 && !vlcIntel64bit && cancelScanDecrypt != 1)
-        {
-            
-            /* we are 64 bit */
-            
-            /* Appropriate VLC not found, so cancel */
-            cancelScanDecrypt = 1;
-            [self writeToActivityLog: "This version of HandBrake is 64 bit, 64 bit version of vlc not found, scan cancelled"];
-            /*On Screen Notification*/
-            int status;
-            NSBeep();
-            status = NSRunAlertPanel(@"This version of HandBrake is 64 bit, VLC found but not 64 bit!",@"", @"Cancel Scan", @"Attempt Scan Anyway", @"Get 64 bit VLC", nil);
-            [NSApp requestUserAttention:NSCriticalRequest];
-            
-            if (status == NSAlertDefaultReturn)
-            {
-                /* User chose to cancel the scan */
-                [self writeToActivityLog: "cannot open physical dvd VLC found but not 64 bit, scan cancelled"];
-                cancelScanDecrypt = 1;
-            }
-            else if (status == NSAlertAlternateReturn)
-            {
-                [self writeToActivityLog: "user overrode 64-bit warning trying to open physical dvd without proper decryption"];
-                cancelScanDecrypt = 0;
-            }
-            else
-            {
-                /* User chose to go download vlc (as they rightfully should) so we send them to the vlc site */
-                [[NSWorkspace sharedWorkspace] openURL:[NSURL URLWithString:@"http://www.videolan.org/vlc/download-macosx.html"]];
-            }
-            
-        }    
-        else if (vlcFound && hb_arch == 32 && !vlcIntel32bit && cancelScanDecrypt != 1)
-        {
-            /* we are 32 bit */
-            /* Appropriate VLC not found, so cancel */
-            cancelScanDecrypt = 1;
-            [self writeToActivityLog: "This version of HandBrake is 32 bit, 32 bit version of vlc not found, scan cancelled"];
-            /*On Screen Notification*/
-            int status;
-            NSBeep();
-            status = NSRunAlertPanel(@"This version of HandBrake is 32 bit, VLC found but not 32 bit!",@"", @"Cancel Scan", @"Attempt Scan Anyway", @"Get 32 bit VLC", nil);
-            [NSApp requestUserAttention:NSCriticalRequest];
-            
-            if (status == NSAlertDefaultReturn)
-            {
-                /* User chose to cancel the scan */
-                [self writeToActivityLog: "cannot open physical dvd VLC found but not 32 bit, scan cancelled"];
-                cancelScanDecrypt = 1;
-            }
-            else if (status == NSAlertAlternateReturn)
-            {
-                [self writeToActivityLog: "user overrode 32-bit warning trying to open physical dvd without proper decryption"];
-                cancelScanDecrypt = 0;
-            }
-            else
-            {
-                /* User chose to go download vlc (as they rightfully should) so we send them to the vlc site */
-                [[NSWorkspace sharedWorkspace] openURL:[NSURL URLWithString:@"http://www.videolan.org/vlc/download-macosx.html"]];
-            }
-            
-        } 
     }
     
     if (cancelScanDecrypt == 0)
@@ -2557,10 +2444,9 @@ fWorkingCount = 0;
         [self writeToActivityLog: "trying to open a physical dvd at: %s", [scanPath UTF8String]];
 
         /* lets check for vlc here to make sure we have a dylib available to use for decrypting */
-        NSString *vlcPath = @"/Applications/VLC.app";
-        NSFileManager * fileManager = [NSFileManager defaultManager];
-	    if ([fileManager fileExistsAtPath:vlcPath] == 0) 
-	    {
+        void *dvdcss = dlopen("libdvdcss.2.dylib", RTLD_LAZY);
+        if (dvdcss == NULL) 
+        {
             /*vlc not found in /Applications so we set the bool to cancel scanning to 1 */
             cancelScanDecrypt = 1;
             [self writeToActivityLog: "VLC app not found for decrypting physical dvd"];
@@ -2589,6 +2475,7 @@ fWorkingCount = 0;
         else
         {
             /* VLC was found in /Applications so all is well, we can carry on using vlc's libdvdcss.dylib for decrypting if needed */
+            dlclose(dvdcss);
             [self writeToActivityLog: "VLC app found for decrypting physical dvd"];
         }
     }
diff --git a/macosx/main.mm b/macosx/main.mm
index 5f73a0c60..0d0b01a6a 100644
--- a/macosx/main.mm
+++ b/macosx/main.mm
@@ -5,6 +5,10 @@
    It may be used under the terms of the GNU General Public License. */
 
 #include <Cocoa/Cocoa.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+
 #import "hb.h"
 
 void SigHandler( int signal )
@@ -30,8 +34,115 @@ extern int mm_flags;
 int mm_support();
 }
 
+char * str_printf(const char *fmt, ...)
+{
+    /* Guess we need no more than 100 bytes. */
+    int len;
+    va_list ap;
+    int size = 100;
+    char *tmp, *str = NULL;
+
+    str = (char*)malloc(size);
+    while (1) 
+    {
+        /* Try to print in the allocated space. */
+        va_start(ap, fmt);
+        len = vsnprintf(str, size, fmt, ap);
+        va_end(ap);
+
+        /* If that worked, return the string. */
+        if (len > -1 && len < size) {
+            return str;
+        }
+
+        /* Else try again with more space. */
+        if (len > -1)    /* glibc 2.1 */
+            size = len+1; /* precisely what is needed */
+        else           /* glibc 2.0 */
+            size *= 2;  /* twice the old size */
+
+        tmp = (char*)realloc(str, size);
+        if (tmp == NULL) {
+            return str;
+        }
+        str = tmp;
+    }
+}
+
+#define EXTRA_VLC_DYLD_PATH "/Applications/VLC.app/Contents/MacOS/lib"
+#define DEFAULT_DYLD_PATH "/usr/local/lib:/usr/lib"
+
 int main( int argc, const char ** argv )
 {
+    char *dylib_path;
+    int no_exec = 0;
+
+    // Check for flag that prevents exec bomb.  It
+    // incidentally can be used to prevent adding
+    // our modifications to the dyld env vars.
+    if ( argc > 1 && strncmp(argv[1], "-n", 2) == 0 )
+        no_exec = 1;
+
+    if ( !no_exec )
+    {
+        dylib_path = getenv("DYLD_FALLBACK_LIBRARY_PATH");
+        if ( dylib_path == NULL ||
+             strstr( dylib_path, "/Applications/VLC.app/Contents/MacOS/lib" ) == NULL )
+        {
+            char *path = NULL;
+            char *home;
+            int result = -1;
+
+            home = getenv("HOME");
+
+            if ( dylib_path == NULL )
+            {
+                // Set the system default of $HOME/lib:/usr/local/lib:/usr/lib
+                // And add our extra path
+                if ( home != NULL )
+                {
+                    path = str_printf("%s/lib:%s:%s:%s%s", home, 
+                                      DEFAULT_DYLD_PATH, 
+                                      EXTRA_VLC_DYLD_PATH, 
+                                      home, EXTRA_VLC_DYLD_PATH);
+                }
+                else
+                {
+                    path = str_printf("%s:%s", DEFAULT_DYLD_PATH, EXTRA_VLC_DYLD_PATH);
+                }
+                if ( path != NULL )
+                    result = setenv("DYLD_FALLBACK_LIBRARY_PATH", path, 1);
+            }
+            else
+            {
+                // add our extra path
+                if ( home != NULL )
+                {
+                    path = str_printf("%s:%s:%s%s", dylib_path, EXTRA_VLC_DYLD_PATH,
+                                                        home, EXTRA_VLC_DYLD_PATH);
+                }
+                else
+                {
+                    path = str_printf("%s:%s", dylib_path, EXTRA_VLC_DYLD_PATH);
+                }
+                if ( path != NULL )
+                    result = setenv("DYLD_FALLBACK_LIBRARY_PATH", path, 1);
+            }
+            if ( result == 0 )
+            {
+                const char ** new_argv;
+                int i;
+
+                new_argv = (const char**)malloc( (argc + 2) * sizeof(char*) );
+                new_argv[0] = argv[0];
+                new_argv[1] = "-n";
+                for (i = 1; i < argc; i++)
+                    new_argv[i+1] = argv[i];
+                new_argv[i+1] = NULL;
+                execv(new_argv[0], (char* const*)new_argv);
+            }
+        }
+    }
     mm_flags = mm_support();
     signal( SIGINT, SigHandler );
     hb_register_error_handler(&hb_error_handler);