Merge branch 'windows-unicode'

Fixed unicode filename handling on Windows.
Made all significant projects build with "Unicode" option on Windows.
Fixed unicode string handling in GUI code on all OSes.

From now on: std::string == UTF-8.

Fixed issue 4111.
Fixed issue 5178.
Fixed issue 5980.

Source/Core/Common/Common.vcxproj

@@ -44,7 +44,7 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
     <UseDebugLibraries>false</UseDebugLibraries>
     <ConfigurationType>StaticLibrary</ConfigurationType>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='DebugFast|Win32'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
@@ -54,7 +54,7 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <UseDebugLibraries>false</UseDebugLibraries>
     <ConfigurationType>StaticLibrary</ConfigurationType>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='DebugFast|x64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>

Source/Core/Common/Src/ArmCPUDetect.cpp

@@ -18,6 +18,7 @@
 #include "Common.h"
 #include "CPUDetect.h"
 #include "StringUtil.h"
+#include "FileUtil.h"
 
 const char procfile[] = "/proc/cpuinfo";
 
@@ -27,9 +28,9 @@ char *GetCPUString()
 	char *cpu_string = 0;
 	// Count the number of processor lines in /proc/cpuinfo
 	char buf[1024];
-	FILE *fp;
 
-	fp = fopen(procfile, "r");
+	File::IOFile file(procfile, "r");
+	auto const fp = file.GetHandle();
 	if (!fp)
 		return 0;
 	
@@ -47,9 +48,9 @@ bool CheckCPUFeature(const char *feature)
 {
 	const char marker[] = "Features\t: ";
 	char buf[1024];
-	FILE *fp;
-
-	fp = fopen(procfile, "r");
+	
+	File::IOFile file(procfile, "r");
+	auto const fp = file.GetHandle();
 	if (!fp)
 		return 0;
 	
@@ -73,9 +74,9 @@ int GetCoreCount()
 	const char marker[] = "processor\t: ";
 	int cores = 0;
 	char buf[1024];
-	FILE *fp;
-
-	fp = fopen(procfile, "r");
+	
+	File::IOFile file(procfile, "r");
+	auto const fp = file.GetHandle();
 	if (!fp)
 		return 0;
 	

Source/Core/Common/Src/CDUtils.cpp

@@ -6,6 +6,7 @@
 #include <memory> // for std::unique_ptr
 #ifdef _WIN32
 #include <windows.h>
+#include "StringUtil.h"
 #elif __APPLE__
 #include <CoreFoundation/CoreFoundation.h>
 #include <IOKit/IOBSD.h>
@@ -25,7 +26,7 @@
 
 #ifdef _WIN32
 // takes a root drive path, returns true if it is a cdrom drive
-bool is_cdrom(const char drive[])
+bool is_cdrom(const TCHAR* drive)
 {
 	return (DRIVE_CDROM == GetDriveType(drive));
 }
@@ -36,15 +37,15 @@ std::vector<std::string> cdio_get_devices()
 	std::vector<std::string> drives;
 
 	const DWORD buffsize = GetLogicalDriveStrings(0, NULL);
-	std::unique_ptr<char[]> buff(new char[buffsize]);
-	if (GetLogicalDriveStrings(buffsize, buff.get()) == buffsize - 1)
+	std::vector<TCHAR> buff(buffsize);
+	if (GetLogicalDriveStrings(buffsize, buff.data()) == buffsize - 1)
 	{
-		const char* drive = buff.get();
+		auto drive = buff.data();
 		while (*drive)
 		{
 			if (is_cdrom(drive))
 			{
-				std::string str(drive);
+				std::string str(TStrToUTF8(drive));
 				str.pop_back();	// we don't want the final backslash
 				drives.push_back(std::move(str));
 			}

Source/Core/Common/Src/ConsoleListener.cpp

@@ -61,7 +61,7 @@ void ConsoleListener::Open(bool Hidden, int Width, int Height, const char *Title
 		// Save the window handle that AllocConsole() created
 		hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
 		// Set the console window title
-		SetConsoleTitle(Title);
+		SetConsoleTitle(UTF8ToTStr(Title).c_str());
 		// Set letter space
 		LetterSpace(80, 4000);
 		//MoveWindow(GetConsoleWindow(), 200,200, 800,800, true);
@@ -195,7 +195,7 @@ void ConsoleListener::PixelSpace(int Left, int Top, int Width, int Height, bool
 
 	static const int MAX_BYTES = 1024 * 16;
 
-	std::vector<std::array<CHAR, MAX_BYTES>> Str;
+	std::vector<std::array<TCHAR, MAX_BYTES>> Str;
 	std::vector<std::array<WORD, MAX_BYTES>> Attr;
 
 	// ReadConsoleOutputAttribute seems to have a limit at this level

Source/Core/Common/Src/Crypto/md5.cpp

@@ -37,6 +37,8 @@
 #include <string.h>
 #include <stdio.h>
 
+#include "../FileUtil.h"
+
 /*
  * 32-bit integer manipulation macros (little endian)
  */
@@ -301,7 +303,10 @@ int md5_file( char *path, unsigned char output[16] )
     md5_context ctx;
     unsigned char buf[1024];
 
-    if( ( f = fopen( path, "rb" ) ) == NULL )
+	File::IOFile file(path, "rb");
+	f = file.GetHandle();
+
+    if (f == NULL)
         return( 1 );
 
     md5_starts( &ctx );
@@ -315,11 +320,9 @@ int md5_file( char *path, unsigned char output[16] )
 
     if( ferror( f ) != 0 )
     {
-        fclose( f );
         return( 2 );
     }
 
-    fclose( f );
     return( 0 );
 }
 

Source/Core/Common/Src/Crypto/sha1.cpp

@@ -36,6 +36,8 @@
 #include <string.h>
 #include <stdio.h>
 
+#include "../FileUtil.h"
+
 /*
  * 32-bit integer manipulation macros (big endian)
  */
@@ -335,7 +337,10 @@ int sha1_file( char *path, unsigned char output[20] )
     sha1_context ctx;
     unsigned char buf[1024];
 
-    if( ( f = fopen( path, "rb" ) ) == NULL )
+	File::IOFile file(path, "rb");
+	f = file.GetHandle();
+
+    if (f == NULL)
         return( 1 );
 
     sha1_starts( &ctx );
@@ -349,11 +354,9 @@ int sha1_file( char *path, unsigned char output[20] )
 
     if( ferror( f ) != 0 )
     {
-        fclose( f );
         return( 2 );
     }
 
-    fclose( f );
     return( 0 );
 }
 

Source/Core/Common/Src/ExtendedTrace.cpp

@@ -17,6 +17,7 @@
 #include <windows.h>
 #include <stdio.h>
 #include "ExtendedTrace.h"
+#include "StringUtil.h"
 using namespace std;
 
 #include <tchar.h>
@@ -274,13 +275,21 @@ static BOOL GetSourceInfoFromAddress( UINT address, LPTSTR lpszSourceInfo )
 	return ret;
 }
 
-void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file )
+void PrintFunctionAndSourceInfo(FILE* file, const STACKFRAME& callstack)
+{
+	TCHAR symInfo[BUFFERSIZE] = _T("?");
+	TCHAR srcInfo[BUFFERSIZE] = _T("?");
+
+	GetFunctionInfoFromAddresses((ULONG)callstack.AddrPC.Offset, (ULONG)callstack.AddrFrame.Offset, symInfo);
+	GetSourceInfoFromAddress((ULONG)callstack.AddrPC.Offset, srcInfo);
+	etfprint(file, "     " + TStrToUTF8(srcInfo) + " : " + TStrToUTF8(symInfo) + "\n");
+}
+
+void StackTrace( HANDLE hThread, const char* lpszMessage, FILE *file )
 {
 	STACKFRAME     callStack;
 	BOOL           bResult;
 	CONTEXT        context;
-	TCHAR          symInfo[BUFFERSIZE] = _T("?");
-	TCHAR          srcInfo[BUFFERSIZE] = _T("?");
 	HANDLE         hProcess = GetCurrentProcess();
 
 	// If it's not this thread, let's suspend it, and resume it at the end
@@ -318,9 +327,7 @@ void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file )
 		etfprint(file, "Call stack info: \n");
 		etfprint(file, lpszMessage);
 
-		GetFunctionInfoFromAddresses( (ULONG)callStack.AddrPC.Offset, (ULONG)callStack.AddrFrame.Offset, symInfo );
-		GetSourceInfoFromAddress( (ULONG)callStack.AddrPC.Offset, srcInfo );
-		etfprint(file, string("     ") + srcInfo + string(" : ") + symInfo + string("\n"));
+		PrintFunctionAndSourceInfo(file, callStack);
 
 		for( ULONG index = 0; ; index++ ) 
 		{
@@ -341,9 +348,7 @@ void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file )
 			if( !bResult || callStack.AddrFrame.Offset == 0 ) 
 				break;
 
-			GetFunctionInfoFromAddresses( (ULONG)callStack.AddrPC.Offset, (ULONG)callStack.AddrFrame.Offset, symInfo );
-			GetSourceInfoFromAddress( (UINT)callStack.AddrPC.Offset, srcInfo );
-			etfprint(file, string("     ") + srcInfo + string(" : ") + symInfo + string("\n"));
+			PrintFunctionAndSourceInfo(file, callStack);
 
 		}
 
@@ -351,19 +356,7 @@ void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file )
 			ResumeThread( hThread );
 }
 
-void StackTrace( HANDLE hThread,  wchar_t const*lpszMessage, FILE *file, DWORD eip, DWORD esp, DWORD ebp )
-{
-	// TODO: remove when Common builds as unicode
-	size_t origsize = wcslen(lpszMessage) + 1;
-	const size_t newsize = 100;
-	size_t convertedChars = 0;
-	char nstring[newsize];
-	wcstombs_s(&convertedChars, nstring, origsize, lpszMessage, _TRUNCATE);
-
-	StackTrace(hThread, nstring, file, eip, esp, ebp );
-}
-
-void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file, DWORD eip, DWORD esp, DWORD ebp )
+void StackTrace(HANDLE hThread, const char* lpszMessage, FILE *file, DWORD eip, DWORD esp, DWORD ebp )
 {
 	STACKFRAME     callStack;
 	BOOL           bResult;
@@ -391,9 +384,7 @@ void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file, DWORD eip, DWO
 		etfprint(file, "Call stack info: \n");
 		etfprint(file, lpszMessage);
 
-		GetFunctionInfoFromAddresses( (ULONG)callStack.AddrPC.Offset, (ULONG)callStack.AddrFrame.Offset, symInfo );
-		GetSourceInfoFromAddress( (UINT)callStack.AddrPC.Offset, srcInfo );
-		etfprint(file, string("     ") + srcInfo + string(" : ") + symInfo + string("\n"));
+		PrintFunctionAndSourceInfo(file, callStack);
 
 		for( ULONG index = 0; ; index++ ) 
 		{
@@ -414,10 +405,7 @@ void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file, DWORD eip, DWO
 			if( !bResult || callStack.AddrFrame.Offset == 0 ) 
 				break;
 
-			GetFunctionInfoFromAddresses( (ULONG)callStack.AddrPC.Offset, (ULONG)callStack.AddrFrame.Offset, symInfo );
-			GetSourceInfoFromAddress( (UINT)callStack.AddrPC.Offset, srcInfo );
-			etfprint(file, string("     ") + srcInfo + string(" : ") + symInfo + string("\n"));
-
+			PrintFunctionAndSourceInfo(file, callStack);
 		}
 
 		if ( hThread != GetCurrentThread() )
@@ -426,7 +414,8 @@ void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file, DWORD eip, DWO
 
 char g_uefbuf[2048];
 
-void etfprintf(FILE *file, const char *format, ...) {
+void etfprintf(FILE *file, const char *format, ...)
+{
 	va_list ap;
 	va_start(ap, format);
 	int len = vsprintf(g_uefbuf, format, ap);
@@ -434,7 +423,8 @@ void etfprintf(FILE *file, const char *format, ...) {
 	va_end(ap);
 }
 
-void etfprint(FILE *file, const std::string &text) {
+void etfprint(FILE *file, const std::string &text)
+{
 	size_t len = text.length();
 	fwrite(text.data(), 1, len, file);
 }

Source/Core/Common/Src/ExtendedTrace.h

@@ -26,15 +26,14 @@
 
 #define EXTENDEDTRACEINITIALIZE( IniSymbolPath )	InitSymInfo( IniSymbolPath )
 #define EXTENDEDTRACEUNINITIALIZE()			        UninitSymInfo()
-#define STACKTRACE(file)							StackTrace( GetCurrentThread(), _T(""), file)
-#define STACKTRACE2(file, eip, esp, ebp) StackTrace(GetCurrentThread(), _T(""), file, eip, esp, ebp)
+#define STACKTRACE(file)							StackTrace( GetCurrentThread(), "", file)
+#define STACKTRACE2(file, eip, esp, ebp) StackTrace(GetCurrentThread(), "", file, eip, esp, ebp)
 // class File;
 
 BOOL InitSymInfo( PCSTR );
 BOOL UninitSymInfo();
-void StackTrace( HANDLE, LPCTSTR, FILE *file);
-void StackTrace( HANDLE, LPCTSTR, FILE *file, DWORD eip, DWORD esp, DWORD ebp);
-void StackTrace( HANDLE hThread, wchar_t const* lpszMessage, FILE *file, DWORD eip, DWORD esp, DWORD ebp);
+void StackTrace(HANDLE, char const* msg, FILE *file);
+void StackTrace(HANDLE, char const* msg, FILE *file, DWORD eip, DWORD esp, DWORD ebp);
 
 // functions by Masken
 void etfprintf(FILE *file, const char *format, ...);

Source/Core/Common/Src/FileSearch.cpp

@@ -51,7 +51,7 @@ void CFileSearch::FindFiles(const std::string& _searchString, const std::string&
 	BuildCompleteFilename(GCMSearchPath, _strPath, _searchString);
 #ifdef _WIN32
 	WIN32_FIND_DATA findData;
-	HANDLE FindFirst = FindFirstFile(GCMSearchPath.c_str(), &findData);
+	HANDLE FindFirst = FindFirstFile(UTF8ToTStr(GCMSearchPath).c_str(), &findData);
 
 	if (FindFirst != INVALID_HANDLE_VALUE)
 	{
@@ -62,7 +62,7 @@ void CFileSearch::FindFiles(const std::string& _searchString, const std::string&
 			if (findData.cFileName[0] != '.')
 			{
 				std::string strFilename;
-				BuildCompleteFilename(strFilename, _strPath, findData.cFileName);
+				BuildCompleteFilename(strFilename, _strPath, TStrToUTF8(findData.cFileName));
 				m_FileNames.push_back(strFilename);
 			}
 
@@ -112,7 +112,6 @@ void CFileSearch::FindFiles(const std::string& _searchString, const std::string&
 #endif
 }
 
-
 const CFileSearch::XStringVector& CFileSearch::GetFileNames() const
 {
 	return m_FileNames;

Source/Core/Common/Src/FileUtil.cpp

@@ -41,10 +41,11 @@
 #include <CoreFoundation/CFBundle.h>
 #endif
 
-#include <fstream>
 #include <algorithm>
 #include <sys/stat.h>
 
+#include "StringUtil.h"
+
 #ifndef S_ISDIR
 #define S_ISDIR(m)  (((m)&S_IFMT) == S_IFDIR)
 #endif
@@ -81,7 +82,11 @@ bool Exists(const std::string &filename)
 	std::string copy(filename);
 	StripTailDirSlashes(copy);
 
+#ifdef _WIN32
+	int result = _tstat64(UTF8ToTStr(copy).c_str(), &file_info);
+#else
 	int result = stat64(copy.c_str(), &file_info);
+#endif
 
 	return (result == 0);
 }
@@ -94,7 +99,11 @@ bool IsDirectory(const std::string &filename)
 	std::string copy(filename);
 	StripTailDirSlashes(copy);
 
+#ifdef _WIN32
+	int result = _tstat64(UTF8ToTStr(copy).c_str(), &file_info);
+#else
 	int result = stat64(copy.c_str(), &file_info);
+#endif
 
 	if (result < 0) {
 		WARN_LOG(COMMON, "IsDirectory: stat failed on %s: %s", 
@@ -127,7 +136,7 @@ bool Delete(const std::string &filename)
 	}
 
 #ifdef _WIN32
-	if (!DeleteFile(filename.c_str()))
+	if (!DeleteFile(UTF8ToTStr(filename).c_str()))
 	{
 		WARN_LOG(COMMON, "Delete: DeleteFile failed on %s: %s", 
 				 filename.c_str(), GetLastErrorMsg());
@@ -149,7 +158,7 @@ bool CreateDir(const std::string &path)
 {
 	INFO_LOG(COMMON, "CreateDir: directory %s", path.c_str());
 #ifdef _WIN32
-	if (::CreateDirectory(path.c_str(), NULL))
+	if (::CreateDirectory(UTF8ToTStr(path).c_str(), NULL))
 		return true;
 	DWORD error = GetLastError();
 	if (error == ERROR_ALREADY_EXISTS)
@@ -228,7 +237,7 @@ bool DeleteDir(const std::string &filename)
 	}
 
 #ifdef _WIN32
-	if (::RemoveDirectory(filename.c_str()))
+	if (::RemoveDirectory(UTF8ToTStr(filename).c_str()))
 		return true;
 #else
 	if (rmdir(filename.c_str()) == 0)
@@ -257,7 +266,7 @@ bool Copy(const std::string &srcFilename, const std::string &destFilename)
 	INFO_LOG(COMMON, "Copy: %s --> %s", 
 			srcFilename.c_str(), destFilename.c_str());
 #ifdef _WIN32
-	if (CopyFile(srcFilename.c_str(), destFilename.c_str(), FALSE))
+	if (CopyFile(UTF8ToTStr(srcFilename).c_str(), UTF8ToTStr(destFilename).c_str(), FALSE))
 		return true;
 
 	ERROR_LOG(COMMON, "Copy: failed %s --> %s: %s", 
@@ -342,8 +351,13 @@ u64 GetSize(const std::string &filename)
 		WARN_LOG(COMMON, "GetSize: failed %s: is a directory", filename.c_str());
 		return 0;
 	}
+	
 	struct stat64 buf;
+#ifdef _WIN32
+	if (_tstat64(UTF8ToTStr(filename).c_str(), &buf) == 0)
+#else
 	if (stat64(filename.c_str(), &buf) == 0)
+#endif
 	{
 		DEBUG_LOG(COMMON, "GetSize: %s: %lld",
 				filename.c_str(), (long long)buf.st_size);
@@ -391,13 +405,13 @@ bool CreateEmptyFile(const std::string &filename)
 {
 	INFO_LOG(COMMON, "CreateEmptyFile: %s", filename.c_str()); 
 
-	FILE *pFile = fopen(filename.c_str(), "wb");
-	if (!pFile) {
+	if (!File::IOFile(filename, "wb"))
+	{
 		ERROR_LOG(COMMON, "CreateEmptyFile: failed %s: %s",
 				  filename.c_str(), GetLastErrorMsg());
 		return false;
 	}
-	fclose(pFile);
+
 	return true;
 }
 
@@ -413,7 +427,7 @@ u32 ScanDirectoryTree(const std::string &directory, FSTEntry& parentEntry)
 	// Find the first file in the directory.
 	WIN32_FIND_DATA ffd;
 
-	HANDLE hFind = FindFirstFile((directory + "\\*").c_str(), &ffd);
+	HANDLE hFind = FindFirstFile(UTF8ToTStr(directory + "\\*").c_str(), &ffd);
 	if (hFind == INVALID_HANDLE_VALUE)
 	{
 		FindClose(hFind);
@@ -423,7 +437,7 @@ u32 ScanDirectoryTree(const std::string &directory, FSTEntry& parentEntry)
 	do
 	{
 		FSTEntry entry;
-		const std::string virtualName(ffd.cFileName);
+		const std::string virtualName(TStrToUTF8(ffd.cFileName));
 #else
 	struct dirent dirent, *result = NULL;
 
@@ -480,7 +494,7 @@ bool DeleteDirRecursively(const std::string &directory)
 #ifdef _WIN32
 	// Find the first file in the directory.
 	WIN32_FIND_DATA ffd;
-	HANDLE hFind = FindFirstFile((directory + "\\*").c_str(), &ffd);
+	HANDLE hFind = FindFirstFile(UTF8ToTStr(directory + "\\*").c_str(), &ffd);
 
 	if (hFind == INVALID_HANDLE_VALUE)
 	{
@@ -491,7 +505,7 @@ bool DeleteDirRecursively(const std::string &directory)
 	// windows loop
 	do
 	{
-		const std::string virtualName = ffd.cFileName;
+		const std::string virtualName(TStrToUTF8(ffd.cFileName));
 #else
 	struct dirent dirent, *result = NULL;
 	DIR *dirp = opendir(directory.c_str());
@@ -622,14 +636,14 @@ std::string GetBundleDirectory()
 #endif
 
 #ifdef _WIN32
-std::string &GetExeDirectory()
+std::string& GetExeDirectory()
 {
 	static std::string DolphinPath;
 	if (DolphinPath.empty())
 	{
-		char Dolphin_exe_Path[2048];
-		GetModuleFileNameA(NULL, Dolphin_exe_Path, 2048);
-		DolphinPath = Dolphin_exe_Path;
+		TCHAR Dolphin_exe_Path[2048];
+		GetModuleFileName(NULL, Dolphin_exe_Path, 2048);
+		DolphinPath = TStrToUTF8(Dolphin_exe_Path);
 		DolphinPath = DolphinPath.substr(0, DolphinPath.find_last_of('\\'));
 	}
 	return DolphinPath;
@@ -730,31 +744,19 @@ std::string &GetUserPath(const unsigned int DirIDX, const std::string &newPath)
 
 bool WriteStringToFile(bool text_file, const std::string &str, const char *filename)
 {
-	FILE *f = fopen(filename, text_file ? "w" : "wb");
-	if (!f)
-		return false;
-	size_t len = str.size();
-	if (len != fwrite(str.data(), 1, str.size(), f))	// TODO: string::data() may not be contiguous
-	{
-		fclose(f);
-		return false;
-	}
-	fclose(f);
-	return true;
+	return File::IOFile(filename, text_file ? "w" : "wb").WriteBytes(str.data(), str.size());
 }
 
 bool ReadFileToString(bool text_file, const char *filename, std::string &str)
 {
-	FILE *f = fopen(filename, text_file ? "r" : "rb");
+	File::IOFile file(filename, text_file ? "r" : "rb");
+	auto const f = file.GetHandle();
+
 	if (!f)
 		return false;
-	size_t len = (size_t)GetSize(f);
-	char *buf = new char[len + 1];
-	buf[fread(buf, 1, len, f)] = 0;
-	str = std::string(buf, len);
-	fclose(f);
-	delete [] buf;
-	return true;
+
+	str.resize(GetSize(f));
+	return file.ReadArray(&str[0], str.size());
 }
 
 IOFile::IOFile()
@@ -798,7 +800,7 @@ bool IOFile::Open(const std::string& filename, const char openmode[])
 {
 	Close();
 #ifdef _WIN32
-	fopen_s(&m_file, filename.c_str(), openmode);
+	_tfopen_s(&m_file, UTF8ToTStr(filename).c_str(), UTF8ToTStr(openmode).c_str());
 #else
 	m_file = fopen(filename.c_str(), openmode);
 #endif

Source/Core/Common/Src/FileUtil.h

@@ -25,6 +25,7 @@
 #include <string.h>
 
 #include "Common.h"
+#include "StringUtil.h"
 
 // User directory indices for GetUserPath
 enum {
@@ -226,4 +227,15 @@ private:
 
 }  // namespace
 
+// To deal with Windows being dumb at unicode:
+template <typename T>
+void OpenFStream(T& fstream, const std::string& filename, std::ios_base::openmode openmode)
+{
+#ifdef _WIN32
+	fstream.open(UTF8ToTStr(filename).c_str(), openmode);
+#else
+	fstream.open(filename.c_str(), openmode);
+#endif
+}
+
 #endif

Source/Core/Common/Src/IniFile.cpp

@@ -25,6 +25,7 @@
 #include <fstream>
 #include <algorithm>
 
+#include "FileUtil.h"
 #include "StringUtil.h"
 #include "IniFile.h"
 
@@ -400,7 +401,7 @@ bool IniFile::Load(const char* filename)
 
 	// Open file
 	std::ifstream in;
-	in.open(filename, std::ios::in);
+	OpenFStream(in, filename, std::ios::in);
 
 	if (in.fail()) return false;
 
@@ -452,7 +453,7 @@ bool IniFile::Load(const char* filename)
 bool IniFile::Save(const char* filename)
 {
 	std::ofstream out;
-	out.open(filename, std::ios::out);
+	OpenFStream(out, filename, std::ios::out);
 
 	if (out.fail())
 	{

Source/Core/Common/Src/LinearDiskCache.h

@@ -74,7 +74,7 @@ public:
 		m_num_entries = 0;
 
 		// try opening for reading/writing
-		m_file.open(filename, ios_base::in | ios_base::out | ios_base::binary);
+		OpenFStream(m_file, filename, ios_base::in | ios_base::out | ios_base::binary);
 
 		m_file.seekg(0, std::ios::end);
 		std::fstream::pos_type end_pos = m_file.tellg();

Source/Core/Common/Src/LogManager.cpp

@@ -186,7 +186,7 @@ void LogContainer::Trigger(LogTypes::LOG_LEVELS level, const char *msg)
 
 FileLogListener::FileLogListener(const char *filename)
 {
-	m_logfile.open(filename, std::ios::app);
+	OpenFStream(m_logfile, filename, std::ios::app);
 	SetEnable(true);
 }
 

Source/Core/Common/Src/Misc.cpp

@@ -31,7 +31,7 @@ const char* GetLastErrorMsg()
 #ifdef _WIN32
 	static __declspec(thread) char err_str[buff_size] = {};
 
-	FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(),
+	FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(),
 		MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
 		err_str, buff_size, NULL);
 #else

Source/Core/Common/Src/MsgHandler.cpp

@@ -106,7 +106,7 @@ bool DefaultMsgHandler(const char* caption, const char* text, bool yes_no, int S
     if (Style == QUESTION) STYLE = MB_ICONQUESTION;
     if (Style == WARNING) STYLE = MB_ICONWARNING;
     
-    return IDYES == MessageBox(0, text, caption, STYLE | (yes_no ? MB_YESNO : MB_OK));
+    return IDYES == MessageBox(0, UTF8ToTStr(text).c_str(), UTF8ToTStr(caption).c_str(), STYLE | (yes_no ? MB_YESNO : MB_OK));
     
 #else
     printf("%s\n", text);

Source/Core/Common/Src/NandPaths.cpp

@@ -86,7 +86,8 @@ bool CheckTitleTIK(u64 _titleID)
 
 static void CreateReplacementFile(std::string &filename)
 {
-	std::ofstream replace(filename.c_str());
+	std::ofstream replace;
+	OpenFStream(replace, filename, std::ios_base::out);
 	replace <<"\" __22__\n";
 	replace << "* __2a__\n";
 	//replace << "/ __2f__\n";
@@ -108,7 +109,8 @@ void ReadReplacements(replace_v& replacements)
 	if (!File::Exists(filename))
 		CreateReplacementFile(filename);
 
-	std::ifstream f(filename.c_str());
+	std::ifstream f;
+	OpenFStream(f, filename, std::ios_base::in);
 	char letter;
 	std::string replacement;
 

Source/Core/Common/Src/SDCardUtil.cpp

@@ -29,6 +29,7 @@
 // Modified for Dolphin.
 
 #include "SDCardUtil.h"
+#include "FileUtil.h"
 
 #include <time.h>
 #include <stdio.h>
@@ -190,7 +191,6 @@ bool SDCardCreate(u64 disk_size /*in MB*/, const char* filename)
 {
 	u32 sectors_per_fat;
 	u32 sectors_per_disk;
-	FILE* f;
 
 	// Convert MB to bytes
 	disk_size *= 1024 * 1024;
@@ -207,7 +207,8 @@ bool SDCardCreate(u64 disk_size /*in MB*/, const char* filename)
 	boot_sector_init(s_boot_sector, s_fsinfo_sector, disk_size, nullptr);
 	fat_init(s_fat_head);
 
-	f = fopen(filename, "wb");
+	File::IOFile file(filename, "wb");
+	FILE* const f = file.GetHandle();
 	if (!f)
 	{
 		ERROR_LOG(COMMON, "Could not create file '%s', aborting...\n", filename);
@@ -247,13 +248,11 @@ bool SDCardCreate(u64 disk_size /*in MB*/, const char* filename)
 
 	if (write_empty(f, sectors_per_disk - RESERVED_SECTORS - 2*sectors_per_fat)) goto FailWrite;
 
-	fclose(f);
 	return true;
 
 FailWrite:
 	ERROR_LOG(COMMON, "Could not write to '%s', aborting...\n", filename);
 	if (unlink(filename) < 0)
 		ERROR_LOG(COMMON, "unlink(%s) failed\n%s", filename, GetLastErrorMsg());
-	fclose(f);
 	return false;
 }

Source/Core/Common/Src/StringUtil.cpp

@@ -17,11 +17,21 @@
 
 #include <stdlib.h>
 #include <stdio.h>
+#include <algorithm>
 
 #include "Common.h"
 #include "CommonPaths.h"
 #include "StringUtil.h"
 
+#ifdef _WIN32
+	#include <Windows.h>
+#elif defined(ANDROID)
+
+#else
+	#include <iconv.h>
+	#include <errno.h>
+#endif
+
 // faster than sscanf
 bool AsciiToHex(const char* _szValue, u32& result)
 {
@@ -375,3 +385,137 @@ std::string UriEncode(const std::string & sSrc)
 	delete [] pStart;
 	return sResult;
 }
+
+#ifdef _WIN32
+
+std::string UTF16ToUTF8(const std::wstring& input)
+{
+	auto const size = WideCharToMultiByte(CP_UTF8, 0, input.data(), input.size(), nullptr, 0, nullptr, nullptr);
+
+	std::string output;
+	output.resize(size);
+
+	if (size != WideCharToMultiByte(CP_UTF8, 0, input.data(), input.size(), &output[0], output.size(), nullptr, nullptr))
+		output.clear();
+
+	return output;
+}
+
+std::wstring CPToUTF16(u32 code_page, const std::string& input)
+{
+	auto const size = MultiByteToWideChar(code_page, 0, input.data(), input.size(), nullptr, 0);
+
+	std::wstring output;
+	output.resize(size);
+
+	if (size != MultiByteToWideChar(code_page, 0, input.data(), input.size(), &output[0], output.size()))
+		output.clear();
+
+	return output;
+}
+
+std::wstring UTF8ToUTF16(const std::string& input)
+{
+	return CPToUTF16(CP_UTF8, input);
+}
+
+std::string SHIFTJISToUTF8(const std::string& input)
+{
+	return UTF16ToUTF8(CPToUTF16(932, input));
+}
+
+std::string CP1252ToUTF8(const std::string& input)
+{
+	return UTF16ToUTF8(CPToUTF16(1252, input));
+}
+
+#else
+
+template <typename T>
+std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
+{
+	std::string result;
+	
+#if defined(ANDROID)
+	result = "Not implemented on Android!";
+	
+#else
+	iconv_t const conv_desc = iconv_open("UTF-8", fromcode);
+	if ((iconv_t)-1 == conv_desc)
+	{
+		ERROR_LOG(COMMON, "Iconv initialization failure [%s]: %s", fromcode, strerror(errno));
+	}
+	else
+	{
+		size_t const in_bytes = sizeof(T) * input.size();
+		size_t const out_buffer_size = 4 * in_bytes;
+		
+		std::string out_buffer;
+		out_buffer.resize(out_buffer_size);
+
+		auto src_buffer = &input[0];
+		size_t src_bytes = in_bytes;
+		auto dst_buffer = &out_buffer[0];
+		size_t dst_bytes = out_buffer.size();
+		
+		while (src_bytes != 0)
+		{
+			size_t const iconv_result = iconv(conv_desc, (char**)(&src_buffer), &src_bytes,
+				&dst_buffer, &dst_bytes);
+			
+			if ((size_t)-1 == iconv_result)
+			{
+				if (EILSEQ == errno || EINVAL == errno)
+				{
+					// Try to skip the bad character
+					if (src_bytes != 0)
+					{
+						--src_bytes;
+						++src_buffer;
+					}
+				}
+				else
+				{
+					ERROR_LOG(COMMON, "iconv failure [%s]: %s", fromcode, strerror(errno));
+					break;
+				}
+			}
+		}
+
+		out_buffer.resize(out_buffer_size - dst_bytes);
+		out_buffer.swap(result);
+		
+		iconv_close(conv_desc);
+	}
+	
+#endif
+	return result;
+}
+
+std::string CP1252ToUTF8(const std::string& input)
+{
+	//return CodeToUTF8("CP1252//TRANSLIT", input);
+	//return CodeToUTF8("CP1252//IGNORE", input);
+	return CodeToUTF8("CP1252", input);
+}
+
+std::string SHIFTJISToUTF8(const std::string& input)
+{
+	//return CodeToUTF8("CP932", input);
+	return CodeToUTF8("SJIS", input);
+}
+
+std::string UTF16ToUTF8(const std::wstring& input)
+{
+	std::string result =
+	//	CodeToUTF8("UCS-2", input);
+	//	CodeToUTF8("UCS-2LE", input);
+	//	CodeToUTF8("UTF-16", input);
+		CodeToUTF8("UTF-16LE", input);
+	
+	// TODO: why is this needed?
+	result.erase(std::remove(result.begin(), result.end(), 0x00), result.end());
+	return result;
+}
+
+#endif

Source/Core/Common/Src/StringUtil.h

@@ -97,4 +97,28 @@ std::string ReplaceAll(std::string result, const std::string& src, const std::st
 std::string UriDecode(const std::string & sSrc);
 std::string UriEncode(const std::string & sSrc);
 
+std::string CP1252ToUTF8(const std::string& str);
+std::string SHIFTJISToUTF8(const std::string& str);
+std::string UTF16ToUTF8(const std::wstring& str);
+
+#ifdef _WIN32
+
+std::wstring UTF8ToUTF16(const std::string& str);
+
+#ifdef _UNICODE
+inline std::string TStrToUTF8(const std::wstring& str)
+{ return UTF16ToUTF8(str); }
+
+inline std::wstring UTF8ToTStr(const std::string& str)
+{ return UTF8ToUTF16(str); }
+#else
+inline std::string TStrToUTF8(const std::string& str)
+{ return str; }
+
+inline std::string UTF8ToTStr(const std::string& str)
+{ return str; }
+#endif
+
+#endif
+
 #endif // _STRINGUTIL_H_