- made the menu's text input handler Unicode capable.

Also make sure that the savegame description remains readable. Unlike in-game text this can be done without double-encoding existing UTF-8.

src/c_console.cpp

@@ -412,14 +412,13 @@ public:
 
 	void AddChar(int character)
 	{
-		uint8_t encoded[5];
 		int size;
-		if (utf8_encode(character, encoded, &size) == 0)
+		auto encoded = MakeUTF8(character, &size);
+		if (*encoded != 0)
 		{
-			encoded[size] = 0;
 			if (Text.IsEmpty())
 			{
-				Text = (char*)encoded;
+				Text = encoded;
 			}
 			else
 			{

src/ct_chat.cpp

@@ -292,9 +292,8 @@ static void CT_AddChar (int c)
 	if (CharLen < QUEUESIZE-2)
 	{
 		int size;
-		uint8_t encode[4];
-		ChatQueue.Pop();
-		if (utf8_encode(c, encode, &size) == 0)
+		auto encode = MakeUTF8(c, &size);
+		if (*encode)
 		{
 			for (int i = 0; i < size; i++)
 			{

src/events.cpp

@@ -41,6 +41,7 @@
 #include "d_net.h"
 #include "g_game.h"
 #include "info.h"
+#include "utf8.h"
 
 EventManager staticEventManager;
 EventManager eventManager;
@@ -1026,7 +1027,7 @@ FUiEvent::FUiEvent(const event_t *ev)
 		break;
 	case EV_GUI_Char:
 		KeyChar = ev->data1;
-		KeyString = FString(char(ev->data1));
+		KeyString = MakeUTF8(ev->data1);
 		IsAlt = !!ev->data2; // only true for Win32, not sure about SDL
 		break;
 	default: // mouse event

src/menu/loadsavemenu.cpp

@@ -185,14 +185,11 @@ void FSavegameManager::ReadSaveStrings()
 					if (arc.OpenReader((const char *)data, info->LumpSize))
 					{
 						int savever = 0;
-						FString engine;
-						FString iwad;
-						FString title;
-
 						arc("Save Version", savever);
-						arc("Engine", engine);
-						arc("Game WAD", iwad);
-						arc("Title", title);
+						FString engine = arc.GetString("Engine");
+						FString iwad = arc.GetString("Game WAD");
+						FString title = arc.GetString("Title");
+
 
 						if (engine.Compare(GAMESIG) != 0 || savever > SAVEVER)
 						{
@@ -473,10 +470,10 @@ unsigned FSavegameManager::ExtractSaveData(int index)
 		FSerializer arc(nullptr);
 		if (arc.OpenReader((const char *)data, info->LumpSize))
 		{
-			FString time, pcomment, comment;
+			FString comment;
 
-			arc("Creation Time", time);
-			arc("Comment", pcomment);
+			FString time = arc.GetString("Creation Time");
+			FString pcomment = arc.GetString("Comment");
 
 			comment = time;
 			if (time.Len() > 0) comment += "\n";

src/scripting/thingdef_data.cpp

@@ -1158,3 +1158,17 @@ DEFINE_ACTION_FUNCTION(FStringStruct, AppendFormat)
 	return 0;
 }
 
+DEFINE_ACTION_FUNCTION(FStringStruct, AppendCharacter)
+{
+	PARAM_SELF_STRUCT_PROLOGUE(FString);
+	PARAM_INT(c);
+	self->AppendCharacter(c);
+	return 0;
+}
+
+DEFINE_ACTION_FUNCTION(FStringStruct, DeleteLastCharacter)
+{
+	PARAM_SELF_STRUCT_PROLOGUE(FString);
+	self->DeleteLastCharacter();
+	return 0;
+}

src/serializer.cpp

@@ -221,6 +221,13 @@ struct FWriter
 		else if (mWriter2) mWriter2->Null();
 	}
 
+	void StringU(const char *k, bool encode)
+	{
+		if (encode) k = StringToUnicode(k);
+		if (mWriter1) mWriter1->String(k);
+		else if (mWriter2) mWriter2->String(k);
+	}
+
 	void String(const char *k)
 	{
 		k = StringToUnicode(k);
@@ -813,7 +820,7 @@ FSerializer &FSerializer::StringPtr(const char *key, const char *&charptr)
 
 //==========================================================================
 //
-//
+// Adds a string literal. This won't get double encoded, like a serialized string.
 //
 //==========================================================================
 
@@ -822,11 +829,33 @@ FSerializer &FSerializer::AddString(const char *key, const char *charptr)
 	if (isWriting())
 	{
 		WriteKey(key);
-		w->String(charptr);
+		w->StringU(MakeUTF8(charptr), false);
 	}
 	return *this;
 }
 
+//==========================================================================
+//
+// Reads back a string without any processing.
+//
+//==========================================================================
+
+const char *FSerializer::GetString(const char *key)
+{
+	auto val = r->FindKey(key);
+	if (val != nullptr)
+	{
+		if (val->IsString())
+		{
+			return val->GetString();
+		}
+		else
+		{
+		}
+	}
+	return nullptr;
+}
+
 //==========================================================================
 //
 //

src/serializer.h

@@ -100,6 +100,7 @@ public:
 	FSerializer &Sprite(const char *key, int32_t &spritenum, int32_t *def);
 	FSerializer &StringPtr(const char *key, const char *&charptr);	// This only retrieves the address but creates no permanent copy of the string unlike the regular char* serializer.
 	FSerializer &AddString(const char *key, const char *charptr);
+	const char *GetString(const char *key);
 	FSerializer &ScriptNum(const char *key, int &num);
 	bool isReading() const
 	{

src/utility/utf8.cpp

@@ -247,3 +247,13 @@ const char *MakeUTF8(const char *outline, int *numchars = nullptr)
 	UTF8String.Push(0);
 	return UTF8String.Data();
 }
+
+const char *MakeUTF8(int codepoint, int *psize)
+{
+	int size = 0;
+	UTF8String.Resize(5);
+	utf8_encode(codepoint, (uint8_t*)UTF8String.Data(), &size);
+	UTF8String[size] = 0;
+	if (psize) *psize = size;
+	return UTF8String.Data();
+}

src/utility/utf8.h

@@ -4,5 +4,6 @@ int utf8_encode(int32_t codepoint, uint8_t *buffer, int *size);
 int utf8_decode(const uint8_t *src, int *size);
 int GetCharFromString(const uint8_t *&string);
 const char *MakeUTF8(const char *outline, int *numchars = nullptr);	// returns a pointer to a static buffer, assuming that its caller will immediately process the result. 
+const char *MakeUTF8(int codepoint, int *psize = nullptr);
 
 extern uint16_t win1252map[];

src/utility/zstring.cpp

@@ -39,6 +39,7 @@
 
 #include "zstring.h"
 #include "v_text.h"
+#include "utf8.h"
 
 FNullStringData FString::NullString =
 {
@@ -475,6 +476,28 @@ FString FString::Mid (size_t pos, size_t numChars) const
 	return FString (Chars + pos, numChars);
 }
 
+void FString::AppendCharacter(int codepoint)
+{
+	(*this) << MakeUTF8(codepoint);
+}
+
+void FString::DeleteLastCharacter()
+{
+	if (Len() == 0) return;
+	auto pos = Len() - 1;
+	while (pos > 0 && Chars[pos] >= 0x80 && Chars[pos] < 0xc0) pos--;
+	if (pos <= 0)
+	{
+		Data()->Release();
+		ResetToNull();
+	}
+	else
+	{
+		Truncate(pos);
+	}
+}
+
+
 long FString::IndexOf (const FString &substr, long startIndex) const
 {
 	return IndexOf (substr.Chars, startIndex);

src/utility/zstring.h

@@ -208,6 +208,9 @@ public:
 	FString Right (size_t numChars) const;
 	FString Mid (size_t pos, size_t numChars = ~(size_t)0) const;
 
+	void AppendCharacter(int codepoint);
+	void DeleteLastCharacter();
+
 	long IndexOf (const FString &substr, long startIndex=0) const;
 	long IndexOf (const char *substr, long startIndex=0) const;
 	long IndexOf (char subchar, long startIndex=0) const;

wadsrc/static/zscript/base.txt

@@ -894,6 +894,8 @@ struct StringStruct native
 	native int ToInt(int base = 0) const;
 	native double ToDouble() const;
 	native void Split(out Array<String> tokens, String delimiter, EmptyTokenType keepEmpty = TOK_KEEPEMPTY) const;
+	native void AppendCharacter(int c);
+	native void DeleteLastCharacter();
 }
 
 class SectorEffect : Thinker native

wadsrc/static/zscript/menu/textentermenu.txt

@@ -49,6 +49,7 @@ class TextEnterMenu : Menu
 	int InputGridX;
 	int InputGridY;
 	bool AllowColors;
+	Font displayFont;
 	
 	//=============================================================================
 	//
@@ -76,6 +77,7 @@ class TextEnterMenu : Menu
 			InputGridY = 0;
 		}
 		AllowColors = allowcolors; // [TP]
+		displayFont = SmallFont;
 	}
 
 	static TextEnterMenu Open(Menu parent, String textbuffer, int maxlen, int sizemode, bool showgrid = false, bool allowcolors = false)
@@ -115,9 +117,9 @@ class TextEnterMenu : Menu
 		{
 			mInputGridOkay = false;
 			if (mEnterString.Length() < mEnterSize &&
-				(mSizeMode == 2/*entering player name*/ || SmallFont.StringWidth(mEnterString) < (mEnterSize-1)*8))
+				(mSizeMode == 2/*entering player name*/ || displayFont.StringWidth(mEnterString) < (mEnterSize-1)*8))
 			{
-				mEnterString.AppendFormat("%c", ev.KeyChar);
+				mEnterString.AppendCharacter(ev.KeyChar);
 			}
 			return true;
 		}
@@ -126,7 +128,7 @@ class TextEnterMenu : Menu
 		{
 			if (mEnterString.Length() > 0)
 			{
-				mEnterString.Truncate(mEnterString.Length() - 1);
+				mEnterString.DeleteLastCharacter();
 			}
 		}
 		else if (ev.Type == UIEvent.Type_KeyDown)
@@ -269,7 +271,7 @@ class TextEnterMenu : Menu
 						}
 					}
 					else if (mEnterString.Length() < mEnterSize &&
-						(mSizeMode == 2/*entering player name*/ || SmallFont.StringWidth(mEnterString) < (mEnterSize-1)*8))
+						(mSizeMode == 2/*entering player name*/ || displayFont.StringWidth(mEnterString) < (mEnterSize-1)*8))
 					{
 						mEnterString = mEnterString .. c;
 					}
@@ -298,7 +300,7 @@ class TextEnterMenu : Menu
 			String InputGridChars = Chars;
 			int cell_width = 18 * CleanXfac;
 			int cell_height = 12 * CleanYfac;
-			int top_padding = cell_height / 2 - SmallFont.GetHeight() * CleanYfac / 2;
+			int top_padding = cell_height / 2 - displayFont.GetHeight() * CleanYfac / 2;
 
 			// Darken the background behind the character grid.
 			screen.Dim(0, 0.8, 0, screen.GetHeight() - INPUTGRID_HEIGHT * cell_height, screen.GetWidth(), INPUTGRID_HEIGHT * cell_height);
@@ -319,7 +321,7 @@ class TextEnterMenu : Menu
 				{
 					int xx = x * cell_width - INPUTGRID_WIDTH * cell_width / 2 + screen.GetWidth() / 2;
 					int ch = InputGridChars.CharCodeAt(y * INPUTGRID_WIDTH + x);
-					int width = SmallFont.GetCharWidth(ch);
+					int width = displayFont.GetCharWidth(ch);
 
 					// The highlighted character is yellow; the rest are dark gray.
 					int colr = (x == InputGridX && y == InputGridY) ? Font.CR_YELLOW : Font.CR_DARKGRAY;
@@ -328,7 +330,7 @@ class TextEnterMenu : Menu
 					if (ch > 32)
 					{
 						// Draw a normal character.
-						screen.DrawChar(SmallFont, colr, xx + cell_width/2 - width*CleanXfac/2, yy + top_padding, ch, DTA_CleanNoMove, true);
+						screen.DrawChar(displayFont, colr, xx + cell_width/2 - width*CleanXfac/2, yy + top_padding, ch, DTA_CleanNoMove, true);
 					}
 					else if (ch == 32)
 					{
@@ -336,7 +338,7 @@ class TextEnterMenu : Menu
 						int x1 = xx + cell_width/2 - width * CleanXfac * 3 / 4;
 						int x2 = x1 + width * 3 * CleanXfac / 2;
 						int y1 = yy + top_padding;
-						int y2 = y1 + SmallFont.GetHeight() * CleanYfac;
+						int y2 = y1 + displayFont.GetHeight() * CleanYfac;
 						screen.Clear(x1, y1, x2, y1+CleanYfac, palcolor);	// top
 						screen.Clear(x1, y2, x2, y2+CleanYfac, palcolor);	// bottom
 						screen.Clear(x1, y1+CleanYfac, x1+CleanXfac, y2, palcolor);	// left
@@ -346,8 +348,8 @@ class TextEnterMenu : Menu
 					{
 						// Draw the backspace and end "characters".
 						String str = ch == 8 ? "BS" : "ED";
-						screen.DrawText(SmallFont, colr,
-							xx + cell_width/2 - SmallFont.StringWidth(str)*CleanXfac/2,
+						screen.DrawText(displayFont, colr,
+							xx + cell_width/2 - displayFont.StringWidth(str)*CleanXfac/2,
 							yy + top_padding, str, DTA_CleanNoMove, true);
 					}
 				}