libhb: make libhb internal character code utf8

This makes libhb expect all strings passed to it to be in utf8 format.
The cli converts the converts from the current code page to utf8.  libhb
converts back to the current code page when necessary for libraries that
expect it.


git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@5798 b64f7644-9d1e-0410-96f1-a4d463321fa5

12 files changed, 230 insertions, 30 deletions

diff --git a/libhb/batch.c b/libhb/batch.c
index 420ac6489..f2a015baa 100644
--- a/libhb/batch.c
+++ b/libhb/batch.c
@@ -24,28 +24,28 @@ struct hb_batch_s
 hb_batch_t * hb_batch_init( char * path )
 {
     hb_batch_t    * d;
-    struct stat     sb;
-    DIR           * dir;
+    hb_stat_t       sb;
+    HB_DIR        * dir;
     struct dirent * entry;
     char          * filename;
 
-    if ( stat( path, &sb ) )
+    if ( hb_stat( path, &sb ) )
         return NULL;
 
     if ( !S_ISDIR( sb.st_mode ) )
         return NULL;
 
-    dir = opendir( path );
+    dir = hb_opendir(path);
     if ( dir == NULL )
         return NULL;
 
     d = calloc( sizeof( hb_batch_t ), 1 );
     d->list_file = hb_list_init();
 
-    while ( (entry = readdir( dir ) ) )
+    while ( (entry = hb_readdir( dir ) ) )
     {
         filename = hb_strdup_printf( "%s" DIR_SEP_STR "%s", path, entry->d_name );
-        if ( stat( filename, &sb ) )
+        if ( hb_stat( filename, &sb ) )
         {
             free( filename );
             continue;
@@ -60,7 +60,7 @@ hb_batch_t * hb_batch_init( char * path )
         hb_list_add( d->list_file, filename );
     }
 
-    closedir( dir );
+    hb_closedir( dir );
     if ( hb_list_count( d->list_file ) == 0 )
     {
         hb_list_close( &d->list_file );
diff --git a/libhb/common.c b/libhb/common.c
index 3d5cf36fb..7087046b6 100644
--- a/libhb/common.c
+++ b/libhb/common.c
@@ -19,6 +19,10 @@
 #include "qsv_common.h"
 #endif
 
+#ifdef SYS_MINGW
+#include <windows.h>
+#endif
+
 /**********************************************************************
  * Global variables
  *********************************************************************/
@@ -2560,6 +2564,21 @@ void hb_valog( hb_debug_level_t level, const char * prefix, const char * log, va
     /* Add the end of line */
     strcat( string, "\n" );
 
+#ifdef SYS_MINGW
+    wchar_t     wstring[2*362]; /* 360 chars + \n + \0 */
+
+    // Convert internal utf8 to "console output code page".
+    //
+    // This is just bizarre windows behavior.  You would expect that
+    // printf would automatically convert a wide character string to
+    // the current "console output code page" when using the "%ls" format
+    // specifier.  But it doesn't... so we must do it.
+    if (!MultiByteToWideChar(CP_UTF8, 0, string, -1, wstring, sizeof(wstring)))
+        return;
+    if (!WideCharToMultiByte(GetConsoleOutputCP(), 0, wstring, -1, string, sizeof(string), NULL, NULL))
+        return;
+#endif
+
     /* Print it */
     fprintf( stderr, "%s", string );
 }
diff --git a/libhb/decsrtsub.c b/libhb/decsrtsub.c
index 74377552e..8b8e5f446 100644
--- a/libhb/decsrtsub.c
+++ b/libhb/decsrtsub.c
@@ -535,9 +535,9 @@ static int decsrtInit( hb_work_object_t * w, hb_job_t * job )
 
         } else {
             memset( &pv->current_entry, 0, sizeof( srt_entry_t ) );
-            
-            pv->file = fopen( w->subtitle->config.src_filename, "r" );
-            
+
+            pv->file = hb_fopen(w->subtitle->config.src_filename, "r");
+
             if( !pv->file )
             {
                 hb_error("Could not open the SRT subtitle file '%s'\n", 
diff --git a/libhb/encavcodec.c b/libhb/encavcodec.c
index f8f9d4796..04125301c 100644
--- a/libhb/encavcodec.c
+++ b/libhb/encavcodec.c
@@ -224,7 +224,7 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
 
         if( job->pass == 1 )
         {
-            pv->file = fopen( filename, "wb" );
+            pv->file = hb_fopen(filename, "wb");
             context->flags |= CODEC_FLAG_PASS1;
         }
         else
@@ -232,7 +232,7 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
             int    size;
             char * log;
 
-            pv->file = fopen( filename, "rb" );
+            pv->file = hb_fopen(filename, "rb");
             fseek( pv->file, 0, SEEK_END );
             size = ftell( pv->file );
             fseek( pv->file, 0, SEEK_SET );
diff --git a/libhb/enctheora.c b/libhb/enctheora.c
index 36fd9e98a..d398f7717 100644
--- a/libhb/enctheora.c
+++ b/libhb/enctheora.c
@@ -51,11 +51,11 @@ int enctheoraInit( hb_work_object_t * w, hb_job_t * job )
         hb_get_tempory_filename( job->h, filename, "theroa.log" );
         if ( job->pass == 1 )
         {
-            pv->file = fopen( filename, "wb" );
+            pv->file = hb_fopen(filename, "wb");
         }
         else
         {
-            pv->file = fopen( filename, "rb" );
+            pv->file = hb_fopen(filename, "rb");
         }
     }
 
diff --git a/libhb/hb.c b/libhb/hb.c
index f6e3cb0e0..570d721df 100644
--- a/libhb/hb.c
+++ b/libhb/hb.c
@@ -665,7 +665,7 @@ int hb_save_preview( hb_handle_t * h, int title, int preview, hb_buffer_t *buf )
     hb_get_tempory_filename( h, filename, "%d_%d_%d",
                              hb_get_instance_id(h), title, preview );
 
-    file = fopen( filename, "wb" );
+    file = hb_fopen(filename, "wb");
     if( !file )
     {
         hb_error( "hb_save_preview: fopen failed (%s)", filename );
@@ -719,7 +719,7 @@ hb_buffer_t * hb_read_preview( hb_handle_t * h, int title_idx, int preview )
     hb_get_tempory_filename( h, filename, "%d_%d_%d",
                              hb_get_instance_id(h), title_idx, preview );
 
-    file = fopen( filename, "rb" );
+    file = hb_fopen(filename, "rb");
     if( !file )
     {
         hb_error( "hb_read_preview: fopen failed (%s)", filename );
diff --git a/libhb/muxcommon.c b/libhb/muxcommon.c
index 9417c707b..91b83a7bb 100644
--- a/libhb/muxcommon.c
+++ b/libhb/muxcommon.c
@@ -346,10 +346,10 @@ void muxClose( hb_work_object_t * w )
         // we're all done muxing -- print final stats and cleanup.
         if( job->pass == 0 || job->pass == 2 )
         {
-            struct stat sb;
+            hb_stat_t sb;
             uint64_t bytes_total, frames_total;
 
-            if( !stat( job->file, &sb ) )
+            if (!hb_stat(job->file, &sb))
             {
                 hb_deep_log( 2, "mux: file size, %"PRId64" bytes", (uint64_t) sb.st_size );
 
diff --git a/libhb/muxmkv.c b/libhb/muxmkv.c
index f96a60e76..0a4cdd951 100644
--- a/libhb/muxmkv.c
+++ b/libhb/muxmkv.c
@@ -119,7 +119,18 @@ static int MKVInit( hb_mux_object_t * m )
 
     track = calloc(1, sizeof(mk_TrackConfig));
 
-    m->file = mk_createWriter(job->file, 1000000, 1);
+    // convert file name to current code page
+    char *path = hb_utf8_to_cp(job->file);
+    if (path == NULL)
+    {
+        hb_error("Could not convert string, out of memory?");
+        job->mux_data = NULL;
+        *job->die = 1;
+        return 0;
+    }
+
+    m->file = mk_createWriter(path, 1000000, 1);
+    free(path);
 
     if( !m->file )
     {
diff --git a/libhb/muxmp4.c b/libhb/muxmp4.c
index bd706d29c..ee082ac8b 100644
--- a/libhb/muxmp4.c
+++ b/libhb/muxmp4.c
@@ -20,6 +20,8 @@ struct hb_mux_object_s
 
     hb_job_t * job;
 
+    /* output file name in current code page */
+    char * path;
     /* libmp4v2 handle */
     MP4FileHandle file;
 
@@ -105,17 +107,25 @@ static int MP4Init( hb_mux_object_t * m )
         TRACK_IN_POSTER  = 0x8
     };
 
+    m->path = hb_utf8_to_cp(job->file);
+    if (m->path == NULL)
+    {
+        hb_error("Could not convert string, out of memory?");
+        *job->die = 1;
+        return 0;
+    }
+
     /* Create an empty mp4 file */
     if (job->largeFileSize)
     /* Use 64-bit MP4 file */
     {
-        m->file = MP4Create( job->file, MP4_DETAILS_ERROR, MP4_CREATE_64BIT_DATA );
+        m->file = MP4Create(m->path, MP4_DETAILS_ERROR, MP4_CREATE_64BIT_DATA);
         hb_deep_log( 2, "muxmp4: using 64-bit MP4 formatting.");
     }
     else
     /* Limit MP4s to less than 4 GB */
     {
-        m->file = MP4Create( job->file, MP4_DETAILS_ERROR, 0 );
+        m->file = MP4Create(m->path, MP4_DETAILS_ERROR, 0);
     }
 
     if (m->file == MP4_INVALID_FILE_HANDLE)
@@ -1167,12 +1177,14 @@ static int MP4End( hb_mux_object_t * m )
         {
             hb_log( "muxmp4: optimizing file" );
             char filename[1024]; memset( filename, 0, 1024 );
-            snprintf( filename, 1024, "%s.tmp", job->file );
-            MP4Optimize( job->file, filename, MP4_DETAILS_ERROR );
-            remove( job->file );
-            rename( filename, job->file );
+            snprintf(filename, 1024, "%s.tmp", m->path);
+            MP4Optimize(m->path, filename, MP4_DETAILS_ERROR);
+            remove(m->path);
+            rename(filename, m->path);
         }
+
     }
+    free(m->path);
 
     return 0;
 }
diff --git a/libhb/ports.c b/libhb/ports.c
index 94bb95227..0f5ed363d 100644
--- a/libhb/ports.c
+++ b/libhb/ports.c
@@ -47,6 +47,9 @@
 #ifdef SYS_MINGW
 #include <pthread.h>
 #include <windows.h>
+#include <wchar.h>
+#include <mbctype.h>
+#include <locale.h>
 #endif
 
 #ifdef SYS_SunOS
@@ -100,6 +103,34 @@ int gettimeofday( struct timeval * tv, struct timezone * tz )
 #endif
 */
 
+// Convert utf8 string to current code page.
+// The internal string representation in hb is utf8. But some
+// libraries (libmkv, and mp4v2) expect filenames in the current
+// code page.  So we must convert.
+char * hb_utf8_to_cp(const char *src)
+{
+    char *dst = NULL;
+
+#if defined( SYS_MINGW )
+    int num_chars = MultiByteToWideChar(CP_UTF8, 0, src, -1, NULL, 0);
+    if (num_chars <= 0)
+        return NULL;
+    wchar_t * tmp = calloc(num_chars, sizeof(wchar_t));
+    MultiByteToWideChar(CP_UTF8, 0, src, -1, tmp, num_chars);
+    int len = WideCharToMultiByte(GetACP(), 0, tmp, num_chars, NULL, 0, NULL, NULL);
+    if (len <= 0)
+        return NULL;
+    dst = calloc(len, sizeof(char));
+    WideCharToMultiByte(GetACP(), 0, tmp, num_chars, dst, len, NULL, NULL);
+    free(tmp);
+#else
+    // Other platforms don't have code pages
+    dst = strdup(src);
+#endif
+
+    return dst;
+}
+
 int hb_dvd_region(char *device, int *region_mask)
 {
 #if defined( DVD_LU_SEND_RPC_STATE ) && defined( DVD_AUTH )
@@ -456,17 +487,119 @@ void hb_get_tempory_filename( hb_handle_t * h, char name[1024],
 }
 
 /************************************************************************
+ * hb_stat
+ ************************************************************************
+ * Wrapper to the real stat, needed to handle utf8 filenames on
+ * windows.
+ ***********************************************************************/
+int hb_stat(const char *path, hb_stat_t *sb)
+{
+#ifdef SYS_MINGW
+    wchar_t path_utf16[MAX_PATH];
+    if (!MultiByteToWideChar(CP_UTF8, 0, path, -1, path_utf16, MAX_PATH))
+        return -1;
+    return _wstat64( path_utf16, sb );
+#else
+    return stat(path, sb);
+#endif
+}
+
+/************************************************************************
+ * hb_fopen
+ ************************************************************************
+ * Wrapper to the real fopen, needed to handle utf8 filenames on
+ * windows.
+ ***********************************************************************/
+FILE * hb_fopen(const char *path, const char *mode)
+{
+#ifdef SYS_MINGW
+    FILE *f;
+    wchar_t path_utf16[MAX_PATH];
+    wchar_t mode_utf16[16];
+    if (!MultiByteToWideChar(CP_UTF8, 0, path, -1, path_utf16, MAX_PATH))
+        return NULL;
+    if (!MultiByteToWideChar(CP_UTF8, 0, mode, -1, mode_utf16, 16))
+        return NULL;
+    errno_t ret = _wfopen_s(&f, path_utf16, mode_utf16);
+    if (ret)
+        return NULL;
+    return f;
+#else
+    return fopen(path, mode);
+#endif
+}
+
+HB_DIR* hb_opendir(char *path)
+{
+#ifdef SYS_MINGW
+    HB_DIR *dir;
+    wchar_t path_utf16[MAX_PATH];
+
+    if (!MultiByteToWideChar(CP_UTF8, 0, path, -1, path_utf16, MAX_PATH))
+        return NULL;
+    dir = malloc(sizeof(HB_DIR));
+    if (dir == NULL)
+        return NULL;
+    dir->wdir = _wopendir(path_utf16);
+    if (dir->wdir == NULL)
+    {
+        free(dir);
+        return NULL;
+    }
+    return dir;
+#else
+    return opendir(path);
+#endif
+}
+
+int hb_closedir(HB_DIR *dir)
+{
+#ifdef SYS_MINGW
+    int ret;
+
+    ret = _wclosedir(dir->wdir);
+    free(dir);
+    return ret;
+#else
+    return closedir(dir);
+#endif
+}
+
+struct dirent * hb_readdir(HB_DIR *dir)
+{
+#ifdef SYS_MINGW
+    struct _wdirent *entry;
+    entry = _wreaddir(dir->wdir);
+    if (entry == NULL)
+        return NULL;
+
+    int len = WideCharToMultiByte(CP_UTF8, 0, entry->d_name, -1,
+                                  dir->entry.d_name, sizeof(dir->entry.d_name),
+                                  NULL, NULL );
+    dir->entry.d_ino = entry->d_ino;
+    dir->entry.d_reclen = entry->d_reclen;
+    dir->entry.d_namlen = len - 1;
+    return &dir->entry;
+#else
+    return readdir(dir);
+#endif
+}
+
+/************************************************************************
  * hb_mkdir
  ************************************************************************
  * Wrapper to the real mkdir, needed only because it doesn't take a
  * second argument on Win32. Grrr.
  ***********************************************************************/
-void hb_mkdir( char * name )
+int hb_mkdir(char * path)
 {
 #ifdef SYS_MINGW
-    mkdir( name );
+    wchar_t path_utf16[MAX_PATH];
+    if (!MultiByteToWideChar(CP_UTF8, 0, path, -1, path_utf16, MAX_PATH))
+        return -1;
+    return _wmkdir(path_utf16);
 #else
-    mkdir( name, 0755 );
+    return mkdir(path, 0755);
 #endif
 }
 
diff --git a/libhb/ports.h b/libhb/ports.h
index a87b7ebd6..5021a5176 100644
--- a/libhb/ports.h
+++ b/libhb/ports.h
@@ -47,8 +47,34 @@ int      hb_platform_init();
 char *strtok_r(char *s, const char *delim, char **save_ptr);
 #endif
 
+#ifdef SYS_MINGW
+typedef struct
+{
+    _WDIR *wdir;
+    struct dirent entry;
+} HB_DIR;
+#else
+typedef DIR HB_DIR;
+#endif
+
+#ifdef SYS_MINGW
+typedef struct _stat64 hb_stat_t;
+#else
+typedef struct stat hb_stat_t;
+#endif
+
+HB_DIR* hb_opendir(char *path);
+int hb_closedir(HB_DIR *dir);
+struct dirent * hb_readdir(HB_DIR *dir);
+int hb_mkdir(char * name);
+int hb_stat(const char *path, hb_stat_t *sb);
+FILE * hb_fopen(const char *path, const char *mode);
+
 #ifdef __LIBHB__
 
+// Convert utf8 string to current code page.
+char * hb_utf8_to_cp(const char *src);
+
 /* Everything from now is only used internally and hidden to the UI */
 
 /************************************************************************
@@ -62,7 +88,6 @@ int hb_dvd_region(char *device, int *region_mask);
 void hb_get_temporary_directory( char path[512] );
 void hb_get_tempory_filename( hb_handle_t *, char name[1024],
                               char * fmt, ... );
-void hb_mkdir( char * name );
 
 /************************************************************************
  * Threads
diff --git a/libhb/stream.c b/libhb/stream.c
index 3512b2d93..8aabdff2c 100644
--- a/libhb/stream.c
+++ b/libhb/stream.c
@@ -816,7 +816,7 @@ static void prune_streams(hb_stream_t *d)
  **********************************************************************/
 hb_stream_t * hb_stream_open( char *path, hb_title_t *title, int scan )
 {
-    FILE *f = fopen( path, "rb" );
+    FILE *f = hb_fopen(path, "rb");
     if ( f == NULL )
     {
         hb_log( "hb_stream_open: open %s failed", path );