- added some UTF-8 utilities to FString.

This deprecated CharAt and CharCodeAt for being unsuitable for text processing and in the case of CharCodeAt also for being buggy.
A new replacement, ByteAt has been added that reads a string byte by byte, as well as CodePointCount, which counts the amount of Unicode code points in a string and GetNextCodePoint which reads the string code point by code point.
Note that while this woll work as intended with the currently supported languages as a means to read single characters, there is no guarantee that this will remain so if Unicode support gets extended to things which break the "one code point == one character" assumption.

src/scripting/vmthunks.cpp

@@ -176,6 +176,19 @@ DEFINE_ACTION_FUNCTION_NATIVE(FStringStruct, CharCodeAt, StringCharCodeAt)
 	ACTION_RETURN_INT(StringCharCodeAt(self, pos));
 }
 
+static int StringByteAt(FString *self, int pos)
+{
+	if ((unsigned)pos >= self->Len()) return 0;
+	else return (uint8_t)((*self)[pos]);
+}
+
+DEFINE_ACTION_FUNCTION_NATIVE(FStringStruct, ByteAt, StringByteAt)
+{
+	PARAM_SELF_STRUCT_PROLOGUE(FString);
+	PARAM_INT(pos);
+	ACTION_RETURN_INT(StringByteAt(self, pos));
+}
+
 static void StringFilter(FString *self, FString *result)
 {
 	*result = strbin1(*self);
@@ -288,6 +301,34 @@ DEFINE_ACTION_FUNCTION_NATIVE(FStringStruct, Split, StringSplit)
 	return 0;
 }
 
+static int StringCodePointCount(FString *self)
+{
+	return (int)self->CharacterCount();
+}
+
+DEFINE_ACTION_FUNCTION_NATIVE(FStringStruct, CodePointCount, StringCodePointCount)
+{
+	PARAM_SELF_STRUCT_PROLOGUE(FString);
+	ACTION_RETURN_INT(StringCodePointCount(self));
+}
+
+static int StringNextCodePoint(FString *self, int inposition, int *position)
+{
+	int codepoint = self->GetNextCharacter(inposition);
+	if (position) *position = inposition;
+	return codepoint;
+}
+
+DEFINE_ACTION_FUNCTION_NATIVE(FStringStruct, GetNextCodePoint, StringNextCodePoint)
+{
+	PARAM_SELF_STRUCT_PROLOGUE(FString);
+	PARAM_INT(pos);
+	if (numret > 0) ret[0].SetInt(self->GetNextCharacter(pos));
+	if (numret > 1) ret[1].SetInt(pos);
+	return numret;
+}
+
+
 //=====================================================================================
 //
 // sector_t exports

wadsrc/static/zscript/base.zs

@@ -898,8 +898,9 @@ struct StringStruct native
 	native String Mid(int pos = 0, int len = 2147483647) const;
 	native void Truncate(int newlen);
 	native void Remove(int index, int remlen);
-	native String CharAt(int pos) const;
+	deprecated("4.1") native String CharAt(int pos) const;
-	native int CharCodeAt(int pos) const;
+	deprecated("4.1") native int CharCodeAt(int pos) const;
+	native int ByteAt(int pos) const;
 	native String Filter();
 	native int IndexOf(String substr, int startIndex = 0) const;
 	deprecated("3.5.1") native int LastIndexOf(String substr, int endIndex = 2147483647) const;
@@ -911,6 +912,8 @@ struct StringStruct native
 	native void Split(out Array<String> tokens, String delimiter, EmptyTokenType keepEmpty = TOK_KEEPEMPTY) const;
 	native void AppendCharacter(int c);
 	native void DeleteLastCharacter();
+	native int CodePointCount();
+	native int, int GetNextCodePoint(int position);
 }
 
 class SectorEffect : Thinker native

wadsrc/static/zscript/ui/menu/listmenuitems.zs

@@ -274,7 +274,7 @@ class ListMenuItemTextItem : ListMenuItemSelectable
 		mFont = desc.mFont;
 		mColor = desc.mFontColor;
 		mColorSelected = desc.mFontcolor2;
-		mHotkey = hotkey.CharCodeAt(0);
+		mHotkey = hotkey.GetNextCodePoint(0);
 	}
 	
 	void InitDirect(double x, double y, int height, String hotkey, String text, Font font, int color, int color2, Name child, int param = 0)
@@ -284,7 +284,8 @@ class ListMenuItemTextItem : ListMenuItemSelectable
 		mFont = font;
 		mColor = color;
 		mColorSelected = color2;
-		mHotkey = hotkey.CharCodeAt(0);
+		int pos = 0;
+		mHotkey = hotkey.GetNextCodePoint(0);
 	}
 	
 	override void Drawer(bool selected)
@@ -313,14 +314,14 @@ class ListMenuItemPatchItem : ListMenuItemSelectable
 	void Init(ListMenuDescriptor desc, TextureID patch, String hotkey, Name child, int param = 0)
 	{
 		Super.Init(desc.mXpos, desc.mYpos, desc.mLinespacing, child, param);
-		mHotkey = hotkey.CharCodeAt(0);
+		mHotkey = hotkey.GetNextCodePoint(0);
 		mTexture = patch;
 	}
 	
 	void InitDirect(double x, double y, int height, TextureID patch, String hotkey, Name child, int param = 0)
 	{
 		Super.Init(x, y, height, child, param);
-		mHotkey = hotkey.CharCodeAt(0);
+		mHotkey = hotkey.GetNextCodePoint(0);
 		mTexture = patch;
 	}
 	

wadsrc/static/zscript/ui/menu/textentermenu.zs

@@ -271,7 +271,7 @@ class TextEnterMenu : Menu
 			case MKEY_Enter:
 				if (mInputGridOkay)
 				{
-					int ch = InputGridChars.CharCodeAt(InputGridX + InputGridY * INPUTGRID_WIDTH);
+					int ch = InputGridChars.ByteAt(InputGridX + InputGridY * INPUTGRID_WIDTH);
 					if (ch == 0)			// end
 					{
 						if (mEnterString.Length() > 0)
@@ -338,7 +338,7 @@ class TextEnterMenu : Menu
 				for (int x = 0; x < INPUTGRID_WIDTH; ++x)
 				{
 					int xx = x * cell_width - INPUTGRID_WIDTH * cell_width / 2 + screen.GetWidth() / 2;
-					int ch = InputGridChars.CharCodeAt(y * INPUTGRID_WIDTH + x);
+					int ch = InputGridChars.ByteAt(y * INPUTGRID_WIDTH + x);
 					int width = displayFont.GetCharWidth(ch);
 
 					// The highlighted character is yellow; the rest are dark gray.

wadsrc/static/zscript/ui/statscreen/statscreen.zs

@@ -306,7 +306,7 @@ class StatusScreen abstract play version("2.5")
 		{
 			// Digits are centered in a box the width of the '3' character.
 			// Other characters (specifically, '-') are right-aligned in their cell.
-			int c = text.CharCodeAt(text_p);
+			int c = text.ByteAt(text_p);
 			if (c >= "0" && c <= "9")
 			{
 				x -= fntwidth;

wadsrc/static/zscript/ui/statusbar/alt_hud.zs

@@ -124,7 +124,7 @@ class AltHud ui
 		x += zerowidth / 2;
 		for(int i=0; i < text.length(); i++)
 		{
-			int c = text.CharCodeAt(i);
+			int c = text.ByteAt(i);
 			int width = fnt.GetCharWidth(c);
 			double offset = fnt.GetBottomAlignOffset(c);