- use a wide string for the console input buffer.

Since this needs to do cursor positioning calculations it's the one spot in the entire engine where UTF-8 would simply be to messy, especially when having to deal with double wide characters.

src/c_console.cpp

@@ -164,13 +164,13 @@ struct History
 struct FCommandBuffer
 {
 private:
-	FString Text;	// The actual command line text
+	std::u32string Text;
 	unsigned CursorPos = 0;
 	unsigned StartPos = 0;	// First character to display
-	unsigned CursorPosChars = 0;
+	unsigned CursorPosCells = 0;
-	unsigned StartPosChars = 0;
+	unsigned StartPosCells = 0;
 
-	FString YankBuffer;	// Deleted text buffer
+	std::u32string YankBuffer;	// Deleted text buffer
 
 public:
 	bool AppendToYankBuffer = false;	// Append consecutive deletes to buffer
@@ -186,12 +186,14 @@ public:
 
 	FString GetText() const
 	{
-		return Text;
+		FString build;
+		for (auto chr : Text) build.AppendCharacter(chr);
+		return build;
 	}
 
 	size_t TextLength() const
 	{
-		return Text.Len();
+		return Text.length();
 	}
 
 	void Draw(int x, int y, int scale, bool cursor)
@@ -205,7 +207,7 @@ public:
 			if (cursor)
 			{
 				screen->DrawChar(CurrentConsoleFont, CR_YELLOW,
-					x + CurrentConsoleFont->GetCharWidth(0x1c) + (CursorPosChars - StartPosChars) * CurrentConsoleFont->GetCharWidth(0xb),
+					x + CurrentConsoleFont->GetCharWidth(0x1c) + (CursorPosCells - StartPosCells) * CurrentConsoleFont->GetCharWidth(0xb),
 					y, '\xb', TAG_DONE);
 			}
 		}
@@ -225,7 +227,7 @@ public:
 			if (cursor)
 			{
 				screen->DrawChar(CurrentConsoleFont, CR_YELLOW,
-					x + CurrentConsoleFont->GetCharWidth(0x1c) + (CursorPosChars - StartPosChars) * CurrentConsoleFont->GetCharWidth(0xb),
+					x + CurrentConsoleFont->GetCharWidth(0x1c) + (CursorPosCells - StartPosCells) * CurrentConsoleFont->GetCharWidth(0xb),
 					y, '\xb',
 					DTA_VirtualWidth, screen->GetWidth() / scale,
 					DTA_VirtualHeight, screen->GetHeight() / scale,
@@ -234,72 +236,57 @@ public:
 		}
 	}
 
-	unsigned BytesForChars(unsigned chars)
-	{
-		unsigned bytes = 0;
-		while (chars > 0)
-		{ 
-			if ((Text[bytes++] & 0xc0) != 0x80) chars--;
-		}
-		return bytes;
-	}
-
 	void MakeStartPosGood()
 	{
-		int n = StartPosChars;
+		int n = StartPos;
 		unsigned cols = ConCols / active_con_scale();
 
-		if (StartPosChars >= Text.CharacterCount())
+		if (StartPos >= Text.length())
 		{ // Start of visible line is beyond end of line
-			n = CursorPosChars - cols + 2;
+			n = CursorPos - cols + 2;
 		}
-		if ((CursorPosChars - StartPosChars) >= cols - 2)
+		if ((CursorPos - StartPos) >= cols - 2)
 		{ // The cursor is beyond the visible part of the line
-			n = CursorPosChars - cols + 2;
+			n = CursorPos - cols + 2;
 		}
-		if (StartPosChars > CursorPosChars)
+		if (StartPos > CursorPos)
 		{ // The cursor is in front of the visible part of the line
-			n = CursorPosChars;
+			n = CursorPos;
 		}
-		StartPosChars = MAX(0, n);
+		StartPos = MAX(0, n);
-		StartPos = BytesForChars(StartPosChars);
+		CursorPosCells = CursorPos;
+		StartPosCells = StartPos;
 	}
 
 	void CursorStart()
 	{
 		CursorPos = 0;
 		StartPos = 0;
-		CursorPosChars = 0;
+		CursorPosCells = 0;
-		StartPosChars = 0;
+		StartPosCells = 0;
 	}
 
 	void CursorEnd()
 	{
-		CursorPos = (unsigned)Text.Len();
+		CursorPos = (unsigned)Text.length();
-		CursorPosChars = (unsigned)Text.CharacterCount();
-		StartPosChars = 0;
 		MakeStartPosGood();
 	}
 
 private:
 	void MoveCursorLeft()
 	{
-		CursorPosChars--;
+		CursorPos--;
-		do CursorPos--;
-		while ((Text[CursorPos] & 0xc0) == 0x80);	// Step back to the last non-continuation byte.
 	}
 
 	void MoveCursorRight()
 	{
-		CursorPosChars++;
+		CursorPos++;
-		do CursorPos++;
-		while ((Text[CursorPos] & 0xc0) == 0x80);	// Step back to the last non-continuation byte.
 	}
 
 public:
 	void CursorLeft()
 	{
-		if (CursorPosChars > 0)
+		if (CursorPos > 0)
 		{
 			MoveCursorLeft();
 			MakeStartPosGood();
@@ -308,7 +295,7 @@ public:
 
 	void CursorRight()
 	{
-		if (CursorPosChars < Text.CharacterCount())
+		if (CursorPos < Text.length())
 		{
 			MoveCursorRight();
 			MakeStartPosGood();
@@ -317,20 +304,20 @@ public:
 
 	void CursorWordLeft()
 	{
-		if (CursorPosChars > 0)
+		if (CursorPos > 0)
 		{
 			do MoveCursorLeft();
-			while (CursorPosChars > 0 && Text[CursorPos - 1] != ' ');
+			while (CursorPos > 0 && Text[CursorPos - 1] != ' ');
 			MakeStartPosGood();
 		}
 	}
 
 	void CursorWordRight()
 	{
-		if (CursorPosChars < Text.CharacterCount())
+		if (CursorPos < Text.length())
 		{
 			do MoveCursorRight();
-			while (CursorPosChars < Text.CharacterCount() && Text[CursorPos] != ' ');
+			while (CursorPos < Text.length() && Text[CursorPos] != ' ');
 			MakeStartPosGood();
 		}
 	}
@@ -339,22 +326,17 @@ public:
 	{
 		if (CursorPos > 0)
 		{
-			auto now = CursorPos;
 			MoveCursorLeft();
-			Text.Remove(CursorPos, now - CursorPos);
+			Text.erase(CursorPos, 1);
 			MakeStartPosGood();
 		}
 	}
 
 	void DeleteRight()
 	{
-		if (CursorPosChars < Text.CharacterCount())
+		if (CursorPos < Text.length())
 		{
-			auto now = CursorPos;
+			Text.erase(CursorPos, 1);
-			MoveCursorRight();
-			Text.Remove(now, CursorPos - now);
-			CursorPos = now;
-			CursorPosChars--;
 			MakeStartPosGood();
 		}
 	}
@@ -368,11 +350,11 @@ public:
 			CursorWordLeft();
 
 			if (AppendToYankBuffer) {
-				YankBuffer = FString(&Text[CursorPos], now - CursorPos) + YankBuffer;
+				YankBuffer = Text.substr(CursorPos, now - CursorPos) + YankBuffer;
 			} else {
-				YankBuffer = FString(&Text[CursorPos], now - CursorPos);
+				YankBuffer = Text.substr(CursorPos, now - CursorPos);
 			}
-			Text.Remove(CursorPos, now - CursorPos);
+			Text.erase(CursorPos, now - CursorPos);
 			MakeStartPosGood();
 		}
 	}
@@ -382,47 +364,41 @@ public:
 		if (CursorPos > 0)
 		{
 			if (AppendToYankBuffer) {
-				YankBuffer = FString(&Text[0], CursorPos) + YankBuffer;
+				YankBuffer = Text.substr(0, CursorPos) + YankBuffer;
 			} else {
-				YankBuffer = FString(&Text[0], CursorPos);
+				YankBuffer = Text.substr(0, CursorPos);
 			}
-			Text.Remove(0, CursorPos);
+			Text.erase(0, CursorPos);
 			CursorStart();
 		}
 	}
 
 	void DeleteLineRight()
 	{
-		if (CursorPos < Text.Len())
+		if (CursorPos < Text.length())
 		{
 			if (AppendToYankBuffer) {
-				YankBuffer += FString(&Text[CursorPos], Text.Len() - CursorPos);
+				YankBuffer += Text.substr(CursorPos, Text.length() - CursorPos);
 			} else {
-				YankBuffer = FString(&Text[CursorPos], Text.Len() - CursorPos);
+				YankBuffer = Text.substr(CursorPos, Text.length() - CursorPos);
 			}
-			Text.Truncate(CursorPos);
+			Text.resize(CursorPos);
 			CursorEnd();
 		}
 	}
 
 	void AddChar(int character)
 	{
-		int size;
+		if (Text.length() == 0)
-		auto encoded = MakeUTF8(character, &size);
-		if (*encoded != 0)
 		{
-			if (Text.IsEmpty())
+			Text += character;
-			{
-				Text = encoded;
-			}
-			else
-			{
-				Text.Insert(CursorPos, (char*)encoded);
-			}
-			CursorPos += size;
-			CursorPosChars++;
-			MakeStartPosGood();
 		}
+		else
+		{
+			Text.insert(CursorPos, 1, character);
+		}
+		CursorPos++;
+		MakeStartPosGood();
 	}
 
 	void AddString(FString clip)
@@ -431,35 +407,52 @@ public:
 		{
 			// Only paste the first line.
 			long brk = clip.IndexOfAny("\r\n\b");
+			std::u32string build;
 			if (brk >= 0)
 			{
 				clip.Truncate(brk);
-				clip = MakeUTF8(clip.GetChars());	// Make sure that we actually have UTF-8 text.
 			}
-			if (Text.IsEmpty())
+			auto strp = (const uint8_t*)clip.GetChars();
+			while (auto chr = GetCharFromString(strp)) build += chr;
+			
+			if (Text.length() == 0)
 			{
-				Text = clip;
+				Text = build;
 			}
 			else
 			{
-				Text.Insert(CursorPos, clip);
+				Text.insert(CursorPos, build);
 			}
-			CursorPos += (unsigned)clip.Len();
+			CursorPos += (unsigned)build.length();
-			CursorPosChars += (unsigned)clip.CharacterCount();
 			MakeStartPosGood();
 		}
 	}
 
 	void SetString(const FString &str)
 	{
-		Text = MakeUTF8(str);
+		Text.clear();
+		auto strp = (const uint8_t*)str.GetChars();
+		while (auto chr = GetCharFromString(strp)) Text += chr;
+
 		CursorEnd();
 		MakeStartPosGood();
 	}
 
 	void AddYankBuffer()
 	{
-		AddString(YankBuffer);
+		if (YankBuffer.length() > 0)
+		{
+			if (Text.length() == 0)
+			{
+				Text = YankBuffer;
+			}
+			else
+			{
+				Text.insert(CursorPos, YankBuffer);
+			}
+			CursorPos += (unsigned)YankBuffer.length();
+			MakeStartPosGood();
+		}
 	}
 };
 static FCommandBuffer CmdLine;

src/g_statusbar/sbarinfo.cpp

@@ -48,6 +48,7 @@
 #include "g_levellocals.h"
 #include "vm.h"
 #include "i_system.h"
+#include "utf8.h"
 
 #define ARTIFLASH_OFFSET (statusBar->invBarOffset+6)
 enum

src/g_statusbar/shared_sbar.cpp

@@ -58,6 +58,7 @@
 #include "gstrings.h"
 #include "events.h"
 #include "g_game.h"
+#include "utf8.h"
 
 #include "../version.h"
 

src/gamedata/fonts/hexfont.cpp

@@ -220,9 +220,7 @@ public:
 			else Chars[i - FirstChar].XMove = spacing;
 
 		}
-		Printf("----------------------------\n");
 		BuildTranslations (luminosity, nullptr, &TranslationParms[1][0], ActiveColors, nullptr);
-		Printf("----------------------------\n");
 	}
 	
 };

src/gamedata/fonts/v_text.cpp

@@ -38,6 +38,7 @@
 #include <wctype.h>
 
 #include "v_text.h"
+#include "utf8.h"
 
 
 #include "v_video.h"

src/gamedata/fonts/v_text.h

@@ -85,6 +85,4 @@ inline TArray<FBrokenLines> V_BreakLines (FFont *font, int maxwidth, const char
 inline TArray<FBrokenLines> V_BreakLines (FFont *font, int maxwidth, const FString &str, bool preservecolor = false)
  { return V_BreakLines (font, maxwidth, (const uint8_t *)str.GetChars(), preservecolor); }
 
-int GetCharFromString(const uint8_t *&string);
-
 #endif //__V_TEXT_H__

src/rendering/2d/v_drawtext.cpp

@@ -38,6 +38,7 @@
 #include <wctype.h>
 
 #include "v_text.h"
+#include "utf8.h"
 
 
 #include "v_video.h"
@@ -248,10 +249,14 @@ DEFINE_ACTION_FUNCTION(_Screen, DrawChar)
 //
 //==========================================================================
 
-void DFrameBuffer::DrawTextCommon(FFont *font, int normalcolor, double x, double y, const char *string, DrawParms &parms)
+// This is only needed as a dummy. The code using wide strings does not need color control.
+EColorRange V_ParseFontColor(const char32_t *&color_value, int normalcolor, int boldcolor) { return CR_UNTRANSLATED; } 
+
+template<class chartype>
+void DFrameBuffer::DrawTextCommon(FFont *font, int normalcolor, double x, double y, const chartype *string, DrawParms &parms)
 {
 	int 		w;
-	const uint8_t *ch;
+	const chartype *ch;
 	int 		c;
 	double 		cx;
 	double 		cy;
@@ -274,13 +279,13 @@ void DFrameBuffer::DrawTextCommon(FFont *font, int normalcolor, double x, double
 
 	kerning = font->GetDefaultKerning();
 
-	ch = (const uint8_t *)string;
+	ch = string;
 	cx = x;
 	cy = y;
 
 
 	auto currentcolor = normalcolor;
-	while ((const char *)ch - string < parms.maxstrlen)
+	while (ch - string < parms.maxstrlen)
 	{
 		c = GetCharFromString(ch);
 		if (!c)
@@ -327,6 +332,24 @@ void DFrameBuffer::DrawText(FFont *font, int normalcolor, double x, double y, co
 	Va_List tags;
 	DrawParms parms;
 
+	if (font == NULL || string == NULL)
+		return;
+
+	va_start(tags.list, tag_first);
+	bool res = ParseDrawTextureTags(nullptr, 0, 0, tag_first, tags, &parms, true);
+	va_end(tags.list);
+	if (!res)
+	{
+		return;
+	}
+	DrawTextCommon(font, normalcolor, x, y, (const uint8_t*)string, parms);
+}
+
+void DFrameBuffer::DrawText(FFont *font, int normalcolor, double x, double y, const char32_t *string, int tag_first, ...)
+{
+	Va_List tags;
+	DrawParms parms;
+
 	if (font == NULL || string == NULL)
 		return;
 
@@ -353,7 +376,7 @@ void DFrameBuffer::DrawText(FFont *font, int normalcolor, double x, double y, co
 	{
 		return;
 	}
-	DrawTextCommon(font, normalcolor, x, y, string, parms);
+	DrawTextCommon(font, normalcolor, x, y, (const uint8_t*)string, parms);
 }
 
 DEFINE_ACTION_FUNCTION(_Screen, DrawText)

src/utility/utf8.h

@@ -3,6 +3,10 @@
 int utf8_encode(int32_t codepoint, uint8_t *buffer, int *size);
 int utf8_decode(const uint8_t *src, int *size);
 int GetCharFromString(const uint8_t *&string);
+inline int GetCharFromString(const char32_t *&string)
+{
+	return *string++;
+}
 const char *MakeUTF8(const char *outline, int *numchars = nullptr);	// returns a pointer to a static buffer, assuming that its caller will immediately process the result. 
 const char *MakeUTF8(int codepoint, int *psize = nullptr);
 

src/v_video.h

@@ -343,7 +343,8 @@ protected:
 
 	template<class T>
 	bool ParseDrawTextureTags(FTexture *img, double x, double y, uint32_t tag, T& tags, DrawParms *parms, bool fortext) const;
-	void DrawTextCommon(FFont *font, int normalcolor, double x, double y, const char *string, DrawParms &parms);
+	template<class T>
+	void DrawTextCommon(FFont *font, int normalcolor, double x, double y, const T *string, DrawParms &parms);
 
 	F2DDrawer m2DDrawer;
 private:
@@ -529,6 +530,7 @@ public:
 	void DrawText(FFont *font, int normalcolor, double x, double y, const char *string, VMVa_List &args);
 	void DrawChar(FFont *font, int normalcolor, double x, double y, int character, int tag_first, ...);
 	void DrawChar(FFont *font, int normalcolor, double x, double y, int character, VMVa_List &args);
+	void DrawText(FFont *font, int normalcolor, double x, double y, const char32_t *string, int tag_first, ...);
 
 	void DrawFrame(int left, int top, int width, int height);
 	void DrawBorder(FTextureID, int x1, int y1, int x2, int y2);

src/win32/i_system.cpp

@@ -70,6 +70,7 @@
 #include "x86.h"
 #include "stats.h"
 #include "v_text.h"
+#include "utf8.h"
 
 #include "d_main.h"
 #include "d_net.h"