From eb4eb1ac0084cd217af1b4ed9ddaaac484a255ec Mon Sep 17 00:00:00 2001 From: Christoph Oelckers Date: Sun, 10 Mar 2019 17:14:34 +0100 Subject: [PATCH] - use a wide string for the console input buffer. Since this needs to do cursor positioning calculations it's the one spot in the entire engine where UTF-8 would simply be to messy, especially when having to deal with double wide characters. --- src/c_console.cpp | 161 +++++++++++++++----------------- src/g_statusbar/sbarinfo.cpp | 1 + src/g_statusbar/shared_sbar.cpp | 1 + src/gamedata/fonts/hexfont.cpp | 2 - src/gamedata/fonts/v_text.cpp | 1 + src/gamedata/fonts/v_text.h | 2 - src/rendering/2d/v_drawtext.cpp | 33 ++++++- src/utility/utf8.h | 4 + src/v_video.h | 4 +- src/win32/i_system.cpp | 1 + 10 files changed, 116 insertions(+), 94 deletions(-) diff --git a/src/c_console.cpp b/src/c_console.cpp index 4bf7d09b9..1e6976544 100644 --- a/src/c_console.cpp +++ b/src/c_console.cpp @@ -164,13 +164,13 @@ struct History struct FCommandBuffer { private: - FString Text; // The actual command line text + std::u32string Text; unsigned CursorPos = 0; unsigned StartPos = 0; // First character to display - unsigned CursorPosChars = 0; - unsigned StartPosChars = 0; + unsigned CursorPosCells = 0; + unsigned StartPosCells = 0; - FString YankBuffer; // Deleted text buffer + std::u32string YankBuffer; // Deleted text buffer public: bool AppendToYankBuffer = false; // Append consecutive deletes to buffer @@ -186,12 +186,14 @@ public: FString GetText() const { - return Text; + FString build; + for (auto chr : Text) build.AppendCharacter(chr); + return build; } size_t TextLength() const { - return Text.Len(); + return Text.length(); } void Draw(int x, int y, int scale, bool cursor) @@ -205,7 +207,7 @@ public: if (cursor) { screen->DrawChar(CurrentConsoleFont, CR_YELLOW, - x + CurrentConsoleFont->GetCharWidth(0x1c) + (CursorPosChars - StartPosChars) * CurrentConsoleFont->GetCharWidth(0xb), + x + CurrentConsoleFont->GetCharWidth(0x1c) + (CursorPosCells - StartPosCells) * CurrentConsoleFont->GetCharWidth(0xb), y, '\xb', TAG_DONE); } } @@ -225,7 +227,7 @@ public: if (cursor) { screen->DrawChar(CurrentConsoleFont, CR_YELLOW, - x + CurrentConsoleFont->GetCharWidth(0x1c) + (CursorPosChars - StartPosChars) * CurrentConsoleFont->GetCharWidth(0xb), + x + CurrentConsoleFont->GetCharWidth(0x1c) + (CursorPosCells - StartPosCells) * CurrentConsoleFont->GetCharWidth(0xb), y, '\xb', DTA_VirtualWidth, screen->GetWidth() / scale, DTA_VirtualHeight, screen->GetHeight() / scale, @@ -234,72 +236,57 @@ public: } } - unsigned BytesForChars(unsigned chars) - { - unsigned bytes = 0; - while (chars > 0) - { - if ((Text[bytes++] & 0xc0) != 0x80) chars--; - } - return bytes; - } - void MakeStartPosGood() { - int n = StartPosChars; + int n = StartPos; unsigned cols = ConCols / active_con_scale(); - if (StartPosChars >= Text.CharacterCount()) + if (StartPos >= Text.length()) { // Start of visible line is beyond end of line - n = CursorPosChars - cols + 2; + n = CursorPos - cols + 2; } - if ((CursorPosChars - StartPosChars) >= cols - 2) + if ((CursorPos - StartPos) >= cols - 2) { // The cursor is beyond the visible part of the line - n = CursorPosChars - cols + 2; + n = CursorPos - cols + 2; } - if (StartPosChars > CursorPosChars) + if (StartPos > CursorPos) { // The cursor is in front of the visible part of the line - n = CursorPosChars; + n = CursorPos; } - StartPosChars = MAX(0, n); - StartPos = BytesForChars(StartPosChars); + StartPos = MAX(0, n); + CursorPosCells = CursorPos; + StartPosCells = StartPos; } void CursorStart() { CursorPos = 0; StartPos = 0; - CursorPosChars = 0; - StartPosChars = 0; + CursorPosCells = 0; + StartPosCells = 0; } void CursorEnd() { - CursorPos = (unsigned)Text.Len(); - CursorPosChars = (unsigned)Text.CharacterCount(); - StartPosChars = 0; + CursorPos = (unsigned)Text.length(); MakeStartPosGood(); } private: void MoveCursorLeft() { - CursorPosChars--; - do CursorPos--; - while ((Text[CursorPos] & 0xc0) == 0x80); // Step back to the last non-continuation byte. + CursorPos--; } void MoveCursorRight() { - CursorPosChars++; - do CursorPos++; - while ((Text[CursorPos] & 0xc0) == 0x80); // Step back to the last non-continuation byte. + CursorPos++; } public: void CursorLeft() { - if (CursorPosChars > 0) + if (CursorPos > 0) { MoveCursorLeft(); MakeStartPosGood(); @@ -308,7 +295,7 @@ public: void CursorRight() { - if (CursorPosChars < Text.CharacterCount()) + if (CursorPos < Text.length()) { MoveCursorRight(); MakeStartPosGood(); @@ -317,20 +304,20 @@ public: void CursorWordLeft() { - if (CursorPosChars > 0) + if (CursorPos > 0) { do MoveCursorLeft(); - while (CursorPosChars > 0 && Text[CursorPos - 1] != ' '); + while (CursorPos > 0 && Text[CursorPos - 1] != ' '); MakeStartPosGood(); } } void CursorWordRight() { - if (CursorPosChars < Text.CharacterCount()) + if (CursorPos < Text.length()) { do MoveCursorRight(); - while (CursorPosChars < Text.CharacterCount() && Text[CursorPos] != ' '); + while (CursorPos < Text.length() && Text[CursorPos] != ' '); MakeStartPosGood(); } } @@ -339,22 +326,17 @@ public: { if (CursorPos > 0) { - auto now = CursorPos; MoveCursorLeft(); - Text.Remove(CursorPos, now - CursorPos); + Text.erase(CursorPos, 1); MakeStartPosGood(); } } void DeleteRight() { - if (CursorPosChars < Text.CharacterCount()) + if (CursorPos < Text.length()) { - auto now = CursorPos; - MoveCursorRight(); - Text.Remove(now, CursorPos - now); - CursorPos = now; - CursorPosChars--; + Text.erase(CursorPos, 1); MakeStartPosGood(); } } @@ -368,11 +350,11 @@ public: CursorWordLeft(); if (AppendToYankBuffer) { - YankBuffer = FString(&Text[CursorPos], now - CursorPos) + YankBuffer; + YankBuffer = Text.substr(CursorPos, now - CursorPos) + YankBuffer; } else { - YankBuffer = FString(&Text[CursorPos], now - CursorPos); + YankBuffer = Text.substr(CursorPos, now - CursorPos); } - Text.Remove(CursorPos, now - CursorPos); + Text.erase(CursorPos, now - CursorPos); MakeStartPosGood(); } } @@ -382,47 +364,41 @@ public: if (CursorPos > 0) { if (AppendToYankBuffer) { - YankBuffer = FString(&Text[0], CursorPos) + YankBuffer; + YankBuffer = Text.substr(0, CursorPos) + YankBuffer; } else { - YankBuffer = FString(&Text[0], CursorPos); + YankBuffer = Text.substr(0, CursorPos); } - Text.Remove(0, CursorPos); + Text.erase(0, CursorPos); CursorStart(); } } void DeleteLineRight() { - if (CursorPos < Text.Len()) + if (CursorPos < Text.length()) { if (AppendToYankBuffer) { - YankBuffer += FString(&Text[CursorPos], Text.Len() - CursorPos); + YankBuffer += Text.substr(CursorPos, Text.length() - CursorPos); } else { - YankBuffer = FString(&Text[CursorPos], Text.Len() - CursorPos); + YankBuffer = Text.substr(CursorPos, Text.length() - CursorPos); } - Text.Truncate(CursorPos); + Text.resize(CursorPos); CursorEnd(); } } void AddChar(int character) { - int size; - auto encoded = MakeUTF8(character, &size); - if (*encoded != 0) + if (Text.length() == 0) { - if (Text.IsEmpty()) - { - Text = encoded; - } - else - { - Text.Insert(CursorPos, (char*)encoded); - } - CursorPos += size; - CursorPosChars++; - MakeStartPosGood(); + Text += character; } + else + { + Text.insert(CursorPos, 1, character); + } + CursorPos++; + MakeStartPosGood(); } void AddString(FString clip) @@ -431,35 +407,52 @@ public: { // Only paste the first line. long brk = clip.IndexOfAny("\r\n\b"); + std::u32string build; if (brk >= 0) { clip.Truncate(brk); - clip = MakeUTF8(clip.GetChars()); // Make sure that we actually have UTF-8 text. } - if (Text.IsEmpty()) + auto strp = (const uint8_t*)clip.GetChars(); + while (auto chr = GetCharFromString(strp)) build += chr; + + if (Text.length() == 0) { - Text = clip; + Text = build; } else { - Text.Insert(CursorPos, clip); + Text.insert(CursorPos, build); } - CursorPos += (unsigned)clip.Len(); - CursorPosChars += (unsigned)clip.CharacterCount(); + CursorPos += (unsigned)build.length(); MakeStartPosGood(); } } void SetString(const FString &str) { - Text = MakeUTF8(str); + Text.clear(); + auto strp = (const uint8_t*)str.GetChars(); + while (auto chr = GetCharFromString(strp)) Text += chr; + CursorEnd(); MakeStartPosGood(); } void AddYankBuffer() { - AddString(YankBuffer); + if (YankBuffer.length() > 0) + { + if (Text.length() == 0) + { + Text = YankBuffer; + } + else + { + Text.insert(CursorPos, YankBuffer); + } + CursorPos += (unsigned)YankBuffer.length(); + MakeStartPosGood(); + } } }; static FCommandBuffer CmdLine; diff --git a/src/g_statusbar/sbarinfo.cpp b/src/g_statusbar/sbarinfo.cpp index 388cdb367..a2d1701fd 100644 --- a/src/g_statusbar/sbarinfo.cpp +++ b/src/g_statusbar/sbarinfo.cpp @@ -48,6 +48,7 @@ #include "g_levellocals.h" #include "vm.h" #include "i_system.h" +#include "utf8.h" #define ARTIFLASH_OFFSET (statusBar->invBarOffset+6) enum diff --git a/src/g_statusbar/shared_sbar.cpp b/src/g_statusbar/shared_sbar.cpp index 4a2db9faf..11920dd7c 100644 --- a/src/g_statusbar/shared_sbar.cpp +++ b/src/g_statusbar/shared_sbar.cpp @@ -58,6 +58,7 @@ #include "gstrings.h" #include "events.h" #include "g_game.h" +#include "utf8.h" #include "../version.h" diff --git a/src/gamedata/fonts/hexfont.cpp b/src/gamedata/fonts/hexfont.cpp index b443df2e7..4567cc324 100644 --- a/src/gamedata/fonts/hexfont.cpp +++ b/src/gamedata/fonts/hexfont.cpp @@ -220,9 +220,7 @@ public: else Chars[i - FirstChar].XMove = spacing; } - Printf("----------------------------\n"); BuildTranslations (luminosity, nullptr, &TranslationParms[1][0], ActiveColors, nullptr); - Printf("----------------------------\n"); } }; diff --git a/src/gamedata/fonts/v_text.cpp b/src/gamedata/fonts/v_text.cpp index dc2729df1..76afe04ae 100644 --- a/src/gamedata/fonts/v_text.cpp +++ b/src/gamedata/fonts/v_text.cpp @@ -38,6 +38,7 @@ #include #include "v_text.h" +#include "utf8.h" #include "v_video.h" diff --git a/src/gamedata/fonts/v_text.h b/src/gamedata/fonts/v_text.h index 30efa286f..90c38d44c 100644 --- a/src/gamedata/fonts/v_text.h +++ b/src/gamedata/fonts/v_text.h @@ -85,6 +85,4 @@ inline TArray V_BreakLines (FFont *font, int maxwidth, const char inline TArray V_BreakLines (FFont *font, int maxwidth, const FString &str, bool preservecolor = false) { return V_BreakLines (font, maxwidth, (const uint8_t *)str.GetChars(), preservecolor); } -int GetCharFromString(const uint8_t *&string); - #endif //__V_TEXT_H__ diff --git a/src/rendering/2d/v_drawtext.cpp b/src/rendering/2d/v_drawtext.cpp index e0bc0d639..12a7f9742 100644 --- a/src/rendering/2d/v_drawtext.cpp +++ b/src/rendering/2d/v_drawtext.cpp @@ -38,6 +38,7 @@ #include #include "v_text.h" +#include "utf8.h" #include "v_video.h" @@ -248,10 +249,14 @@ DEFINE_ACTION_FUNCTION(_Screen, DrawChar) // //========================================================================== -void DFrameBuffer::DrawTextCommon(FFont *font, int normalcolor, double x, double y, const char *string, DrawParms &parms) +// This is only needed as a dummy. The code using wide strings does not need color control. +EColorRange V_ParseFontColor(const char32_t *&color_value, int normalcolor, int boldcolor) { return CR_UNTRANSLATED; } + +template +void DFrameBuffer::DrawTextCommon(FFont *font, int normalcolor, double x, double y, const chartype *string, DrawParms &parms) { int w; - const uint8_t *ch; + const chartype *ch; int c; double cx; double cy; @@ -274,13 +279,13 @@ void DFrameBuffer::DrawTextCommon(FFont *font, int normalcolor, double x, double kerning = font->GetDefaultKerning(); - ch = (const uint8_t *)string; + ch = string; cx = x; cy = y; auto currentcolor = normalcolor; - while ((const char *)ch - string < parms.maxstrlen) + while (ch - string < parms.maxstrlen) { c = GetCharFromString(ch); if (!c) @@ -327,6 +332,24 @@ void DFrameBuffer::DrawText(FFont *font, int normalcolor, double x, double y, co Va_List tags; DrawParms parms; + if (font == NULL || string == NULL) + return; + + va_start(tags.list, tag_first); + bool res = ParseDrawTextureTags(nullptr, 0, 0, tag_first, tags, &parms, true); + va_end(tags.list); + if (!res) + { + return; + } + DrawTextCommon(font, normalcolor, x, y, (const uint8_t*)string, parms); +} + +void DFrameBuffer::DrawText(FFont *font, int normalcolor, double x, double y, const char32_t *string, int tag_first, ...) +{ + Va_List tags; + DrawParms parms; + if (font == NULL || string == NULL) return; @@ -353,7 +376,7 @@ void DFrameBuffer::DrawText(FFont *font, int normalcolor, double x, double y, co { return; } - DrawTextCommon(font, normalcolor, x, y, string, parms); + DrawTextCommon(font, normalcolor, x, y, (const uint8_t*)string, parms); } DEFINE_ACTION_FUNCTION(_Screen, DrawText) diff --git a/src/utility/utf8.h b/src/utility/utf8.h index cb437a2a1..ab4577533 100644 --- a/src/utility/utf8.h +++ b/src/utility/utf8.h @@ -3,6 +3,10 @@ int utf8_encode(int32_t codepoint, uint8_t *buffer, int *size); int utf8_decode(const uint8_t *src, int *size); int GetCharFromString(const uint8_t *&string); +inline int GetCharFromString(const char32_t *&string) +{ + return *string++; +} const char *MakeUTF8(const char *outline, int *numchars = nullptr); // returns a pointer to a static buffer, assuming that its caller will immediately process the result. const char *MakeUTF8(int codepoint, int *psize = nullptr); diff --git a/src/v_video.h b/src/v_video.h index fea29e714..837dd3edb 100644 --- a/src/v_video.h +++ b/src/v_video.h @@ -343,7 +343,8 @@ protected: template bool ParseDrawTextureTags(FTexture *img, double x, double y, uint32_t tag, T& tags, DrawParms *parms, bool fortext) const; - void DrawTextCommon(FFont *font, int normalcolor, double x, double y, const char *string, DrawParms &parms); + template + void DrawTextCommon(FFont *font, int normalcolor, double x, double y, const T *string, DrawParms &parms); F2DDrawer m2DDrawer; private: @@ -529,6 +530,7 @@ public: void DrawText(FFont *font, int normalcolor, double x, double y, const char *string, VMVa_List &args); void DrawChar(FFont *font, int normalcolor, double x, double y, int character, int tag_first, ...); void DrawChar(FFont *font, int normalcolor, double x, double y, int character, VMVa_List &args); + void DrawText(FFont *font, int normalcolor, double x, double y, const char32_t *string, int tag_first, ...); void DrawFrame(int left, int top, int width, int height); void DrawBorder(FTextureID, int x1, int y1, int x2, int y2); diff --git a/src/win32/i_system.cpp b/src/win32/i_system.cpp index f9691146a..4087e1ec9 100644 --- a/src/win32/i_system.cpp +++ b/src/win32/i_system.cpp @@ -70,6 +70,7 @@ #include "x86.h" #include "stats.h" #include "v_text.h" +#include "utf8.h" #include "d_main.h" #include "d_net.h"