From 50ebca20bb3882f535f32de78b0008a8be5df63c Mon Sep 17 00:00:00 2001
From: Christoph Oelckers <coelckers@users.noreply.github.com>
Date: Thu, 14 Mar 2019 21:42:02 +0100
Subject: [PATCH] - do not use iswspace to check raw UTF-8.

At least under MSVC this function reports 0x85 as whitespace, but it is a continuation byte for UTF-8 sequences and may not be treated as whitespace.
---
 src/gamedata/fonts/v_text.cpp | 11 +++++++----
 src/gamedata/stringtable.cpp  |  6 ++++--
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/gamedata/fonts/v_text.cpp b/src/gamedata/fonts/v_text.cpp
index 76afe04ae3..ac176e12aa 100644
--- a/src/gamedata/fonts/v_text.cpp
+++ b/src/gamedata/fonts/v_text.cpp
@@ -75,6 +75,9 @@ TArray<FBrokenLines> V_BreakLines (FFont *font, int maxwidth, const uint8_t *str
 	bool lastWasSpace = false;
 	int kerning = font->GetDefaultKerning ();
 
+	// The real isspace is a bit too badly defined, so use our own one
+	auto myisspace = [](int ch) { return ch == '\t' || ch == '\r' || ch == '\n' || ch == ' '; };
+
 	w = 0;
 
 	while ( (c = GetCharFromString(string)) )
@@ -104,7 +107,7 @@ TArray<FBrokenLines> V_BreakLines (FFont *font, int maxwidth, const uint8_t *str
 			continue;
 		}
 
-		if (iswspace(c)) 
+		if (myisspace(c)) 
 		{
 			if (!lastWasSpace)
 			{
@@ -137,12 +140,12 @@ TArray<FBrokenLines> V_BreakLines (FFont *font, int maxwidth, const uint8_t *str
 			start = space;
 			space = NULL;
 
-			while (*start && iswspace (*start) && *start != '\n')
+			while (*start && myisspace (*start) && *start != '\n')
 				start++;
 			if (*start == '\n')
 				start++;
 			else
-				while (*start && iswspace (*start))
+				while (*start && myisspace (*start))
 					start++;
 			string = start;
 		}
@@ -160,7 +163,7 @@ TArray<FBrokenLines> V_BreakLines (FFont *font, int maxwidth, const uint8_t *str
 		while (s < string)
 		{
 			// If there is any non-white space in the remainder of the string, add it.
-			if (!iswspace (*s++))
+			if (!myisspace (*s++))
 			{
 				auto i = Lines.Reserve(1);
 				breakit (&Lines[i], font, start, string, linecolor);
diff --git a/src/gamedata/stringtable.cpp b/src/gamedata/stringtable.cpp
index 8d3344a0e1..b70727c75e 100644
--- a/src/gamedata/stringtable.cpp
+++ b/src/gamedata/stringtable.cpp
@@ -150,13 +150,15 @@ bool FStringTable::readSheetIntoTable(xlsxioreader reader, const char *sheetname
 		int column = 0;
 		char *value;
 		table.Reserve(1);
+		auto myisspace = [](int ch) { return ch == '\t' || ch == '\r' || ch == '\n' || ch == ' '; };
+
 		while ((value = xlsxioread_sheet_next_cell(sheet)) != nullptr)
 		{
 			auto vcopy = value;
 			if (table.Size() <= (unsigned)row) table.Reserve(1);
-			while (*vcopy && iswspace((unsigned char)*vcopy)) vcopy++;	// skip over leaading whitespace;
+			while (*vcopy && myisspace((unsigned char)*vcopy)) vcopy++;	// skip over leaading whitespace;
 			auto vend = vcopy + strlen(vcopy);
-			while (vend > vcopy && iswspace((unsigned char)vend[-1])) *--vend = 0;	// skip over trailing whitespace
+			while (vend > vcopy && myisspace((unsigned char)vend[-1])) *--vend = 0;	// skip over trailing whitespace
 			ProcessEscapes(vcopy);
 			table[row].Push(vcopy);
 			column++;