From bcf7bc8d34b24c0b7d27c5d93106ca830dda7547 Mon Sep 17 00:00:00 2001 From: Christoph Oelckers Date: Sat, 13 Apr 2019 09:31:36 +0200 Subject: [PATCH] - added some UTF-8 utilities to FString. This deprecated CharAt and CharCodeAt for being unsuitable for text processing and in the case of CharCodeAt also for being buggy. A new replacement, ByteAt has been added that reads a string byte by byte, as well as CodePointCount, which counts the amount of Unicode code points in a string and GetNextCodePoint which reads the string code point by code point. Note that while this woll work as intended with the currently supported languages as a means to read single characters, there is no guarantee that this will remain so if Unicode support gets extended to things which break the "one code point == one character" assumption. --- src/scripting/vmthunks.cpp | 41 +++++++++++++++++++ wadsrc/static/zscript/base.zs | 7 +++- .../static/zscript/ui/menu/listmenuitems.zs | 9 ++-- .../static/zscript/ui/menu/textentermenu.zs | 4 +- .../zscript/ui/statscreen/statscreen.zs | 2 +- wadsrc/static/zscript/ui/statusbar/alt_hud.zs | 2 +- 6 files changed, 55 insertions(+), 10 deletions(-) diff --git a/src/scripting/vmthunks.cpp b/src/scripting/vmthunks.cpp index 42b38edc9..4bf34ede8 100644 --- a/src/scripting/vmthunks.cpp +++ b/src/scripting/vmthunks.cpp @@ -176,6 +176,19 @@ DEFINE_ACTION_FUNCTION_NATIVE(FStringStruct, CharCodeAt, StringCharCodeAt) ACTION_RETURN_INT(StringCharCodeAt(self, pos)); } +static int StringByteAt(FString *self, int pos) +{ + if ((unsigned)pos >= self->Len()) return 0; + else return (uint8_t)((*self)[pos]); +} + +DEFINE_ACTION_FUNCTION_NATIVE(FStringStruct, ByteAt, StringByteAt) +{ + PARAM_SELF_STRUCT_PROLOGUE(FString); + PARAM_INT(pos); + ACTION_RETURN_INT(StringByteAt(self, pos)); +} + static void StringFilter(FString *self, FString *result) { *result = strbin1(*self); @@ -288,6 +301,34 @@ DEFINE_ACTION_FUNCTION_NATIVE(FStringStruct, Split, StringSplit) return 0; } +static int StringCodePointCount(FString *self) +{ + return (int)self->CharacterCount(); +} + +DEFINE_ACTION_FUNCTION_NATIVE(FStringStruct, CodePointCount, StringCodePointCount) +{ + PARAM_SELF_STRUCT_PROLOGUE(FString); + ACTION_RETURN_INT(StringCodePointCount(self)); +} + +static int StringNextCodePoint(FString *self, int inposition, int *position) +{ + int codepoint = self->GetNextCharacter(inposition); + if (position) *position = inposition; + return codepoint; +} + +DEFINE_ACTION_FUNCTION_NATIVE(FStringStruct, GetNextCodePoint, StringNextCodePoint) +{ + PARAM_SELF_STRUCT_PROLOGUE(FString); + PARAM_INT(pos); + if (numret > 0) ret[0].SetInt(self->GetNextCharacter(pos)); + if (numret > 1) ret[1].SetInt(pos); + return numret; +} + + //===================================================================================== // // sector_t exports diff --git a/wadsrc/static/zscript/base.zs b/wadsrc/static/zscript/base.zs index 27f534443..18a80287b 100644 --- a/wadsrc/static/zscript/base.zs +++ b/wadsrc/static/zscript/base.zs @@ -898,8 +898,9 @@ struct StringStruct native native String Mid(int pos = 0, int len = 2147483647) const; native void Truncate(int newlen); native void Remove(int index, int remlen); - native String CharAt(int pos) const; - native int CharCodeAt(int pos) const; + deprecated("4.1") native String CharAt(int pos) const; + deprecated("4.1") native int CharCodeAt(int pos) const; + native int ByteAt(int pos) const; native String Filter(); native int IndexOf(String substr, int startIndex = 0) const; deprecated("3.5.1") native int LastIndexOf(String substr, int endIndex = 2147483647) const; @@ -911,6 +912,8 @@ struct StringStruct native native void Split(out Array tokens, String delimiter, EmptyTokenType keepEmpty = TOK_KEEPEMPTY) const; native void AppendCharacter(int c); native void DeleteLastCharacter(); + native int CodePointCount(); + native int, int GetNextCodePoint(int position); } class SectorEffect : Thinker native diff --git a/wadsrc/static/zscript/ui/menu/listmenuitems.zs b/wadsrc/static/zscript/ui/menu/listmenuitems.zs index 4ed436c75..16b75ad0d 100644 --- a/wadsrc/static/zscript/ui/menu/listmenuitems.zs +++ b/wadsrc/static/zscript/ui/menu/listmenuitems.zs @@ -274,7 +274,7 @@ class ListMenuItemTextItem : ListMenuItemSelectable mFont = desc.mFont; mColor = desc.mFontColor; mColorSelected = desc.mFontcolor2; - mHotkey = hotkey.CharCodeAt(0); + mHotkey = hotkey.GetNextCodePoint(0); } void InitDirect(double x, double y, int height, String hotkey, String text, Font font, int color, int color2, Name child, int param = 0) @@ -284,7 +284,8 @@ class ListMenuItemTextItem : ListMenuItemSelectable mFont = font; mColor = color; mColorSelected = color2; - mHotkey = hotkey.CharCodeAt(0); + int pos = 0; + mHotkey = hotkey.GetNextCodePoint(0); } override void Drawer(bool selected) @@ -313,14 +314,14 @@ class ListMenuItemPatchItem : ListMenuItemSelectable void Init(ListMenuDescriptor desc, TextureID patch, String hotkey, Name child, int param = 0) { Super.Init(desc.mXpos, desc.mYpos, desc.mLinespacing, child, param); - mHotkey = hotkey.CharCodeAt(0); + mHotkey = hotkey.GetNextCodePoint(0); mTexture = patch; } void InitDirect(double x, double y, int height, TextureID patch, String hotkey, Name child, int param = 0) { Super.Init(x, y, height, child, param); - mHotkey = hotkey.CharCodeAt(0); + mHotkey = hotkey.GetNextCodePoint(0); mTexture = patch; } diff --git a/wadsrc/static/zscript/ui/menu/textentermenu.zs b/wadsrc/static/zscript/ui/menu/textentermenu.zs index 6fb150f29..2f179d0d7 100644 --- a/wadsrc/static/zscript/ui/menu/textentermenu.zs +++ b/wadsrc/static/zscript/ui/menu/textentermenu.zs @@ -271,7 +271,7 @@ class TextEnterMenu : Menu case MKEY_Enter: if (mInputGridOkay) { - int ch = InputGridChars.CharCodeAt(InputGridX + InputGridY * INPUTGRID_WIDTH); + int ch = InputGridChars.ByteAt(InputGridX + InputGridY * INPUTGRID_WIDTH); if (ch == 0) // end { if (mEnterString.Length() > 0) @@ -338,7 +338,7 @@ class TextEnterMenu : Menu for (int x = 0; x < INPUTGRID_WIDTH; ++x) { int xx = x * cell_width - INPUTGRID_WIDTH * cell_width / 2 + screen.GetWidth() / 2; - int ch = InputGridChars.CharCodeAt(y * INPUTGRID_WIDTH + x); + int ch = InputGridChars.ByteAt(y * INPUTGRID_WIDTH + x); int width = displayFont.GetCharWidth(ch); // The highlighted character is yellow; the rest are dark gray. diff --git a/wadsrc/static/zscript/ui/statscreen/statscreen.zs b/wadsrc/static/zscript/ui/statscreen/statscreen.zs index 2c592d21c..8c92b8ca7 100644 --- a/wadsrc/static/zscript/ui/statscreen/statscreen.zs +++ b/wadsrc/static/zscript/ui/statscreen/statscreen.zs @@ -306,7 +306,7 @@ class StatusScreen abstract play version("2.5") { // Digits are centered in a box the width of the '3' character. // Other characters (specifically, '-') are right-aligned in their cell. - int c = text.CharCodeAt(text_p); + int c = text.ByteAt(text_p); if (c >= "0" && c <= "9") { x -= fntwidth; diff --git a/wadsrc/static/zscript/ui/statusbar/alt_hud.zs b/wadsrc/static/zscript/ui/statusbar/alt_hud.zs index c74baec74..6c4fbbb3f 100644 --- a/wadsrc/static/zscript/ui/statusbar/alt_hud.zs +++ b/wadsrc/static/zscript/ui/statusbar/alt_hud.zs @@ -124,7 +124,7 @@ class AltHud ui x += zerowidth / 2; for(int i=0; i < text.length(); i++) { - int c = text.CharCodeAt(i); + int c = text.ByteAt(i); int width = fnt.GetCharWidth(c); double offset = fnt.GetBottomAlignOffset(c);