(svn r9358) [0.5] -Backport from trunk (r9055, r9082, r9083, r9084, r9085, r9086):

- Codechange: Change windows unicode handling and allow a pure non-unicode build to function. (r9055)
- Codechange: [win32] Update VS2003 and VS2005 project files to build in UNICODE mode. When making a release it is probably better to make two binaries, one without UNICODE, the other with, guaranteeing full Win9x compatibility (UNICODE with MSLU also works, without it's even better). (r9082)
- Codechange: Be more lenient when trimming UTF-8 strings and don't terminate the string when an invalid encoding is encountered, but only focus on maximum length. (r9083)
- Fix: [win9x] Clipboard paste for Windows95 (doesn't have CF_UNICODETEXT) correctly converts the input to the current locale. (r9084)
- Fix: [win32] Move the initialisation of _codepage (non-UNICODE) to winMain as a dedicated server, or different video driver will not have a win32 messageloop. (r9085)
- Fix: [win32] Rewrite keyboard input and handle all keypresses in a WM_CHAR event. This saves us from doing translation (ToUnicode[Ex], ToAscii[Ex]), and we get free IME-input support as a plus. (r9086)

fios.c

@@ -17,6 +17,7 @@
 #include <sys/stat.h>
 
 #ifdef WIN32
+# include <tchar.h>
 # include <io.h>
 #else
 # include <unistd.h>
@@ -170,8 +171,8 @@ void FiosMakeSavegameName(char *buf, const char *name, size_t size)
 	snprintf(buf, size, "%s" PATHSEP "%s%s", _fios_path, name, extension);
 }
 
-#if defined(WIN32) || defined(WIN64)
+#if defined(WIN32)
-# define unlink _wunlink
+# define unlink _tunlink
 #endif
 
 bool FiosDelete(const char *name)

fios.h

@@ -57,7 +57,7 @@ int CDECL compare_FiosItems(const void *a, const void *b);
 typedef struct DIR DIR;
 
 typedef struct dirent { // XXX - only d_name implemented
-	wchar_t *d_name; /* name of found file */
+	TCHAR *d_name; /* name of found file */
 	/* little hack which will point to parent DIR struct which will
 	 * save us a call to GetFileAttributes if we want information
 	 * about the file (for example in function fio_bla */
@@ -70,14 +70,14 @@ struct DIR {
 	 * note: having only one global instance is not possible because
 	 * multiple independent opendir/readdir sequences must be supported. */
 	dirent ent;
-	WIN32_FIND_DATAW fd;
+	WIN32_FIND_DATA fd;
 	/* since opendir calls FindFirstFile, we need a means of telling the
 	 * first call to readdir that we already have a file.
 	 * that's the case iff this is true */
 	bool at_first_entry;
 };
 
-DIR *opendir(const wchar_t *path);
+DIR *opendir(const TCHAR *path);
 struct dirent *readdir(DIR *d);
 int closedir(DIR *d);
 #else

openttd.vcproj

@@ -54,7 +54,7 @@
 				Name="VCCustomBuildTool"/>
 			<Tool
 				Name="VCLinkerTool"
-				AdditionalDependencies="unicows.lib winmm.lib ws2_32.lib libpng.lib zlibstat.lib dxguid.lib libfreetype2.lib"
+				AdditionalDependencies="winmm.lib ws2_32.lib libpng.lib zlibstat.lib dxguid.lib libfreetype2.lib"
 				OutputFile=".\Release/openttd.exe"
 				LinkIncremental="1"
 				SuppressStartupBanner="TRUE"

openttd_vs80.vcproj

@@ -28,7 +28,7 @@
 			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
 			UseOfMFC="0"
 			ATLMinimizesCRunTimeLibraryUsage="false"
-			CharacterSet="2"
+			CharacterSet="1"
 			WholeProgramOptimization="1"
 			>
 			<Tool
@@ -139,7 +139,7 @@
 			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
 			UseOfMFC="0"
 			ATLMinimizesCRunTimeLibraryUsage="false"
-			CharacterSet="2"
+			CharacterSet="1"
 			WholeProgramOptimization="1"
 			>
 			<Tool
@@ -251,7 +251,7 @@
 			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
 			UseOfMFC="0"
 			ATLMinimizesCRunTimeLibraryUsage="false"
-			CharacterSet="2"
+			CharacterSet="1"
 			>
 			<Tool
 				Name="VCPreBuildEventTool"
@@ -349,7 +349,7 @@
 			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
 			UseOfMFC="0"
 			ATLMinimizesCRunTimeLibraryUsage="false"
-			CharacterSet="2"
+			CharacterSet="1"
 			>
 			<Tool
 				Name="VCPreBuildEventTool"

stdafx.h

@@ -167,9 +167,10 @@
  * call to the same function and is not thread- or reentrancy-safe */
 #if !defined(STRGEN)
 # if defined(WIN32) || defined(WIN64)
-#  define fopen(file, mode) _wfopen(OTTD2FS(file), L ## mode)
+#  include <tchar.h>
-   const char *FS2OTTD(const wchar_t *name);
+#  define fopen(file, mode) _tfopen(OTTD2FS(file), _T(mode))
-   const wchar_t *OTTD2FS(const char *name);
+   const char *FS2OTTD(const TCHAR *name);
+   const TCHAR *OTTD2FS(const char *name);
 # else
 #  define fopen(file, mode) fopen(OTTD2FS(file), mode)
    const char *FS2OTTD(const char *name);

string.c

@@ -283,7 +283,8 @@ size_t Utf8TrimString(char *s, size_t maxlen)
 	const char *ptr = strchr(s, '\0');
 	while (*s != '\0') {
 		size_t len = Utf8EncodedCharLen(*s);
-		if (len == 0) break; // invalid encoding
+		/* Silently ignore invalid UTF8 sequences, our only concern trimming */
+		if (len == 0) len = 1;
 
 		/* Take care when a hard cutoff was made for the string and
 		 * the last UTF8 sequence is invalid */

video/win32_v.c

@@ -36,6 +36,9 @@ bool _window_maximize;
 uint _display_hz;
 uint _fullscreen_bpp;
 static uint16 _bck_resolution[2];
+#if !defined(UNICODE)
+uint _codepage;
+#endif
 
 static void MakePalette(void)
 {
@@ -208,6 +211,8 @@ static void CALLBACK TrackMouseTimerProc(HWND hwnd, UINT msg, UINT event, DWORD
 
 static LRESULT CALLBACK WndProcGdi(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam)
 {
+	static uint32 keycode = 0;
+
 	switch (msg) {
 		case WM_CREATE:
 			SetTimer(hwnd, TID_POLLMOUSE, MOUSE_POLL_DELAY, (TIMERPROC)TrackMouseTimerProc);
@@ -347,33 +352,54 @@ static LRESULT CALLBACK WndProcGdi(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lP
 			return 0;
 		}
 
-		case WM_KEYDOWN: {
+#if !defined(UNICODE)
-			// this is the rewritten ascii input function
+		case WM_INPUTLANGCHANGE: {
-			// it disables windows deadkey handling --> more linux like :D
+			TCHAR locale[6];
-			wchar_t w = 0;
+			LCID lcid = GB(lParam, 0, 16);
-			byte ks[256];
-			uint scancode;
-			uint32 pressed_key;
 
-			GetKeyboardState(ks);
+			int len = GetLocaleInfo(lcid, LOCALE_IDEFAULTANSICODEPAGE, locale, lengthof(locale));
-			if (ToUnicode(wParam, 0, ks, &w, 1, 0) != 1) {
+			if (len != 0) _codepage = _ttoi(locale);
-				/* On win9x ToUnicode always fails, so fall back to ToAscii */
+			return 1;
-				if (ToAscii(wParam, 0, ks, &w, 0) != 1) w = 0; // no translation was possible
+		}
+#endif /* UNICODE */
+
+		case WM_CHAR: {
+			uint scancode = GB(lParam, 16, 8);
+			uint charcode = wParam;
+
+			/* Silently drop all non-text messages as those were handled by WM_KEYDOWN */
+			if (wParam < VK_SPACE) return 0;
+
+#if !defined(UNICODE)
+			{
+				wchar_t w;
+				int len = MultiByteToWideChar(_codepage, 0, (char*)&charcode, 1, &w, 1);
+				charcode = len == 1 ? w : 0;
 			}
+#endif /* UNICODE */
 
-			pressed_key = w | MapWindowsKey(wParam) << 16;
+			/* No matter the keyboard layout, we will map the '~' to the console */
+			scancode = scancode == 41 ? (int)WKC_BACKQUOTE : keycode;
+			HandleKeypress(GB(charcode, 0, 16) | (scancode << 16));
+			return 0;
+		}
 
-			scancode = GB(lParam, 16, 8);
+		case WM_KEYDOWN: {
-			if (scancode == 41) pressed_key = w | WKC_BACKQUOTE << 16;
+			keycode = MapWindowsKey(wParam);
 
-			if (GB(pressed_key, 16, 16) == ('D' | WKC_CTRL) && !_wnd.fullscreen) {
+			/* Silently drop all text messages as those will be handled by WM_CHAR
+			 * WM_KEYDOWN only handles CTRL+ commands and special keys like VK_LEFT, etc. */
+			if (keycode == 0 || (keycode > WKC_PAUSE && GB(keycode, 13, 4) == 0)) return 0;
+
+			if (keycode == ('D' | WKC_CTRL) && !_wnd.fullscreen) {
 				_double_size ^= 1;
 				_wnd.double_size = _double_size;
 				ClientSizeChanged(_wnd.width, _wnd.height);
 				MarkWholeScreenDirty();
 			}
-			HandleKeypress(pressed_key);
+
-			break;
+			HandleKeypress(0 | (keycode << 16));
+			return 0;
 		}
 
 		case WM_SYSKEYDOWN: /* user presses F10 or Alt, both activating the title-menu */
@@ -796,6 +822,7 @@ static void Win32GdiMainLoop(void)
 
 		while (PeekMessage(&mesg, NULL, 0, 0, PM_REMOVE)) {
 			InteractiveRandom(); // randomness
+			TranslateMessage(&mesg);
 			DispatchMessage(&mesg);
 		}
 		if (_exit_game) return;

win32.c

@@ -652,23 +652,23 @@ static inline void dir_free(DIR *d)
 	}
 }
 
-DIR *opendir(const wchar_t *path)
+DIR *opendir(const TCHAR *path)
 {
 	DIR *d;
 	UINT sem = SetErrorMode(SEM_FAILCRITICALERRORS); // disable 'no-disk' message box
-	DWORD fa = GetFileAttributesW(path);
+	DWORD fa = GetFileAttributes(path);
 
 	if ((fa != INVALID_FILE_ATTRIBUTES) && (fa & FILE_ATTRIBUTE_DIRECTORY)) {
 		d = dir_calloc();
 		if (d != NULL) {
-			wchar_t search_path[MAX_PATH];
+			TCHAR search_path[MAX_PATH];
-			bool slash = path[wcslen(path) - 1] == L'\\';
+			bool slash = path[_tcslen(path) - 1] == '\\';
 
 			/* build search path for FindFirstFile, try not to append additional slashes
 			 * as it throws Win9x off its groove for root directories */
-			_snwprintf(search_path, lengthof(search_path), L"%s%s*", path, slash ? L"" : L"\\");
+			_sntprintf(search_path, lengthof(search_path), _T("%s%s*"), path, slash ? _T("") : _T("\\"));
 			*lastof(search_path) = '\0';
-			d->hFind = FindFirstFileW(search_path, &d->fd);
+			d->hFind = FindFirstFile(search_path, &d->fd);
 
 			if (d->hFind != INVALID_HANDLE_VALUE ||
 					GetLastError() == ERROR_NO_MORE_FILES) { // the directory is empty
@@ -699,7 +699,7 @@ struct dirent *readdir(DIR *d)
 		/* the directory was empty when opened */
 		if (d->hFind == INVALID_HANDLE_VALUE) return NULL;
 		d->at_first_entry = false;
-	} else if (!FindNextFileW(d->hFind, &d->fd)) { // determine cause and bail
+	} else if (!FindNextFile(d->hFind, &d->fd)) { // determine cause and bail
 		if (GetLastError() == ERROR_NO_MORE_FILES) SetLastError(prev_err);
 		return NULL;
 	}
@@ -742,7 +742,7 @@ bool FiosIsValidFile(const char *path, const struct dirent *ent, struct stat *sb
 {
 	// hectonanoseconds between Windows and POSIX epoch
 	static const int64 posix_epoch_hns = 0x019DB1DED53E8000LL;
-	const WIN32_FIND_DATAW *fd = &ent->dir->fd;
+	const WIN32_FIND_DATA *fd = &ent->dir->fd;
 
 	sb->st_size  = ((uint64) fd->nFileSizeHigh << 32) + fd->nFileSizeLow;
 	/* UTC FILETIME to seconds-since-1970 UTC
@@ -872,19 +872,22 @@ void ShowInfo(const char *str)
 	int _set_error_mode(int);
 #endif
 
-int APIENTRY _tWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
+int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdShow)
-	LPTSTR lpCmdLine, int nCmdShow)
 {
 	int argc;
 	char *argv[64]; // max 64 command line arguments
 	char *cmdline;
 
+#if !defined(UNICODE)
+	_codepage = GetACP(); // get system codepage as some kind of a default
+#endif /* UNICODE */
+
 #if defined(UNICODE)
 	/* For UNICODE we need to convert the commandline to char* _AND_
 	 * save it because argv[] points into this buffer and thus needs to
 	 * be available between subsequent calls to FS2OTTD() */
 	char cmdlinebuf[MAX_PATH];
-#endif
+#endif /* UNICODE */
 
 	cmdline = WIDE_TO_MB_BUFFER(GetCommandLine(), cmdlinebuf, lengthof(cmdlinebuf));
 
@@ -921,11 +924,17 @@ int APIENTRY _tWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
 void DeterminePaths(void)
 {
 	char *s, *cfg;
-	wchar_t path[MAX_PATH];
 
-	_paths.personal_dir = _paths.game_data_dir = cfg = malloc(MAX_PATH);
+	_paths.personal_dir = _paths.game_data_dir = cfg = (char*)malloc(MAX_PATH);
-	GetCurrentDirectoryW(MAX_PATH - 1, path);
+#if defined(UNICODE)
-	convert_from_fs(path, cfg, MAX_PATH);
+	{
+		TCHAR path[MAX_PATH];
+		GetCurrentDirectory(MAX_PATH - 1, path);
+		convert_from_fs(path, cfg, MAX_PATH);
+	}
+#else
+	GetCurrentDirectory(MAX_PATH - 1, cfg);
+#endif
 
 	cfg[0] = toupper(cfg[0]);
 	s = strchr(cfg, '\0');
@@ -946,10 +955,10 @@ void DeterminePaths(void)
 	_log_file = str_fmt("%sopenttd.log", _paths.personal_dir);
 
 	// make (auto)save and scenario folder
-	CreateDirectoryW(OTTD2FS(_paths.save_dir), NULL);
+	CreateDirectory(OTTD2FS(_paths.save_dir), NULL);
-	CreateDirectoryW(OTTD2FS(_paths.autosave_dir), NULL);
+	CreateDirectory(OTTD2FS(_paths.autosave_dir), NULL);
-	CreateDirectoryW(OTTD2FS(_paths.scenario_dir), NULL);
+	CreateDirectory(OTTD2FS(_paths.scenario_dir), NULL);
-	CreateDirectoryW(OTTD2FS(_paths.heightmap_dir), NULL);
+	CreateDirectory(OTTD2FS(_paths.heightmap_dir), NULL);
 }
 
 /**
@@ -980,14 +989,16 @@ bool InsertTextBufferClipboard(Textbuf *tb)
 		CloseClipboard();
 
 		if (*ret == '\0') return false;
+#if !defined(UNICODE)
 	} else if (IsClipboardFormatAvailable(CF_TEXT)) {
 		OpenClipboard(NULL);
 		cbuf = GetClipboardData(CF_TEXT);
 
 		ptr = GlobalLock(cbuf);
-		ttd_strlcpy(utf8_buf, ptr, lengthof(utf8_buf));
+		ttd_strlcpy(utf8_buf, FS2OTTD(ptr), lengthof(utf8_buf));
 		GlobalUnlock(cbuf);
 		CloseClipboard();
+#endif /* UNICODE */
 	} else {
 		return false;
 	}
@@ -1045,8 +1056,104 @@ int64 GetTS(void)
 	return (__int64)(value * freq);
 }
 
-/** Convert from OpenTTD's encoding to that of the local environment in
+
- * UNICODE. OpenTTD encoding is UTF8, local is wide-char
+/**
+ * Convert to OpenTTD's encoding from that of the local environment.
+ * When the project is built in UNICODE, the system codepage is irrelevant and
+ * the input string is wide. In ANSI mode, the string is in the
+ * local codepage which we'll convert to wide-char, and then to UTF-8.
+ * OpenTTD internal encoding is UTF8.
+ * The returned value's contents can only be guaranteed until the next call to
+ * this function. So if the value is needed for anything else, use convert_from_fs
+ * @param name pointer to a valid string that will be converted (local, or wide)
+ * @return pointer to the converted string; if failed string is of zero-length
+ * @see the current code-page comes from video\win32_v.cpp, event-notification
+ * WM_INPUTLANGCHANGE */
+const char *FS2OTTD(const TCHAR *name)
+{
+	static char utf8_buf[512];
+#if defined(UNICODE)
+	return convert_from_fs(name, utf8_buf, lengthof(utf8_buf));
+#else
+	char *s = utf8_buf;
+
+	for (; *name != '\0'; name++) {
+		wchar_t w;
+		int len = MultiByteToWideChar(_codepage, 0, name, 1, &w, 1);
+		if (len != 1) {
+			DEBUG(misc, 0) ("[utf8] M2W error converting '%c'. Errno %d", *name, GetLastError());
+			continue;
+		}
+
+		if (s + Utf8CharLen(w) >= lastof(utf8_buf)) break;
+		s += Utf8Encode(s, w);
+	}
+
+	*s = '\0';
+	return utf8_buf;
+#endif /* UNICODE */
+}
+
+/**
+ * Convert from OpenTTD's encoding to that of the local environment.
+ * When the project is built in UNICODE the system codepage is irrelevant and
+ * the converted string is wide. In ANSI mode, the UTF8 string is converted
+ * to multi-byte.
+ * OpenTTD internal encoding is UTF8.
+ * The returned value's contents can only be guaranteed until the next call to
+ * this function. So if the value is needed for anything else, use convert_from_fs
+ * @param name pointer to a valid string that will be converted (UTF8)
+ * @return pointer to the converted string; if failed string is of zero-length
+ * @see the current code-page comes from video\win32_v.cpp, event-notification
+ * WM_INPUTLANGCHANGE */
+const TCHAR *OTTD2FS(const char *name)
+{
+	static TCHAR system_buf[512];
+#if defined(UNICODE)
+	return convert_to_fs(name, system_buf, lengthof(system_buf));
+#else
+	char *s = system_buf;
+	WChar c;
+	for (; (c = Utf8Consume(&name)) != '\0';) {
+		char mb;
+		int len;
+
+		if (s >= lastof(system_buf)) break;
+		len = WideCharToMultiByte(_codepage, 0, (wchar_t*)&c, 1, &mb, 1, NULL, NULL);
+		if (len != 1) {
+			DEBUG(misc, 0) ("[utf8] W2M error converting '0x%X'. Errno %d", c, GetLastError());
+			continue;
+		}
+
+		*s++ = mb;
+	}
+
+	*s = '\0';
+	return system_buf;
+#endif /* UNICODE */
+}
+
+
+/** Convert to OpenTTD's encoding from that of the environment in
+ * UNICODE. OpenTTD encoding is UTF8, local is wide
+ * @param name pointer to a valid string that will be converted
+ * @param utf8_buf pointer to a valid buffer that will receive the converted string
+ * @param buflen length in characters of the receiving buffer
+ * @return pointer to utf8_buf. If conversion fails the string is of zero-length */
+char *convert_from_fs(const wchar_t *name, char *utf8_buf, size_t buflen)
+{
+	int len = WideCharToMultiByte(CP_UTF8, 0, name, -1, utf8_buf, buflen, NULL, NULL);
+	if (len == 0) {
+		DEBUG(misc, 0) ("[utf8] W2M error converting wide-string. Errno %d", GetLastError());
+		utf8_buf[0] = '\0';
+	}
+
+	return utf8_buf;
+}
+
+
+/** Convert from OpenTTD's encoding to that of the environment in
+ * UNICODE. OpenTTD encoding is UTF8, local is wide
  * @param name pointer to a valid string that will be converted
  * @param utf16_buf pointer to a valid wide-char buffer that will receive the
  * converted string
@@ -1056,55 +1163,13 @@ wchar_t *convert_to_fs(const char *name, wchar_t *utf16_buf, size_t buflen)
 {
 	int len = MultiByteToWideChar(CP_UTF8, 0, name, -1, utf16_buf, buflen);
 	if (len == 0) {
-		DEBUG(misc, 0) ("[utf8] Error converting '%s'. Errno %d", name, GetLastError());
+		DEBUG(misc, 0) ("[utf8] M2W error converting '%s'. Errno %d", name, GetLastError());
 		utf16_buf[0] = '\0';
 	}
 
 	return utf16_buf;
 }
 
-/** Convert from OpenTTD's encoding to that of the local environment in
- * UNICODE. OpenTTD encoding is UTF8, local is wide-char.
- * The returned value's contents can only be guaranteed until the next call to
- * this function. So if the value is needed for anything else, use convert_from_fs
- * @param name pointer to a valid string that will be converted
- * @return pointer to the converted string; if failed string is of zero-length */
-const wchar_t *OTTD2FS(const char *name)
-{
-	static wchar_t utf16_buf[512];
-	return convert_to_fs(name, utf16_buf, lengthof(utf16_buf));
-}
-
-
-/** Convert to OpenTTD's encoding from that of the local environment in
- * UNICODE. OpenTTD encoding is UTF8, local is wide-char
- * @param name pointer to a valid string that will be converted
- * @param utf8_buf pointer to a valid buffer that will receive the converted string
- * @param buflen length in characters of the receiving buffer
- * @return pointer to utf8_buf. If conversion fails the string is of zero-length */
-char *convert_from_fs(const wchar_t *name, char *utf8_buf, size_t buflen)
-{
-	int len = WideCharToMultiByte(CP_UTF8, 0, name, -1, utf8_buf, buflen, NULL, NULL);
-	if (len == 0) {
-		DEBUG(misc, 0) ("[utf8] Error converting wide-string. Errno %d", GetLastError());
-		utf8_buf[0] = '\0';
-	}
-
-	return utf8_buf;
-}
-
-/** Convert to OpenTTD's encoding from that of the local environment in
- * UNICODE. OpenTTD encoding is UTF8, local is wide-char.
- * The returned value's contents can only be guaranteed until the next call to
- * this function. So if the value is needed for anything else, use convert_from_fs
- * @param name pointer to a valid string that will be converted
- * @return pointer to the converted string; if failed string is of zero-length */
-const char *FS2OTTD(const wchar_t *name)
-{
-	static char utf8_buf[512];
-	return convert_from_fs(name, utf8_buf, lengthof(utf8_buf));
-}
-
 /** Our very own SHGetFolderPath function for support of windows operating
  * systems that don't have this function (eg Win9x, etc.). We try using the
  * native function, and if that doesn't exist we will try a more crude approach

win32.h

@@ -24,6 +24,7 @@ wchar_t *convert_to_fs(const char *name, wchar_t *utf16_buf, size_t buflen);
 # define WIDE_TO_MB(str) FS2OTTD(str)
 # define WIDE_TO_MB_BUFFER(str, buffer, buflen) convert_from_fs(str, buffer, buflen)
 #else
+extern uint _codepage; // local code-page in the system @see win32_v.cpp:WM_INPUTLANGCHANGE
 # define MB_TO_WIDE(str) (str)
 # define MB_TO_WIDE_BUFFER(str, buffer, buflen) (str)
 # define WIDE_TO_MB(str) (str)