- rewrote the language table so that it doesn't have to reload everything on a language change.

It now reads everything into a two-dimensional TMap and creates a list of mappings that apply to the current setting.
The constant need for reloading was the main blocker in redesigning how Dehacked strings get inserted. Currently they override everything, but IWAD-based Dehacked text shouldn't block PWAD overrides from PWADs' LANGUAGE lumps and instead be treated as coming from an [en default] block.

This also renames the main block from [enu default] to [en default], because it should be treated as the English default for all English locales and not just make it fall through to the base default as it did before.

src/d_main.cpp

@@ -2427,7 +2427,7 @@ void D_DoomMain (void)
 		}
 
 		// [RH] Initialize localizable strings.
-		GStrings.LoadStrings (false);
+		GStrings.LoadStrings ();
 
 		V_InitFontColors ();
 

src/doomstat.cpp

@@ -56,10 +56,10 @@ CUSTOM_CVAR (Float, teamdamage, 0.f, CVAR_SERVERINFO)
 	}
 }
 
-CUSTOM_CVAR (String, language, "auto", CVAR_ARCHIVE)
+CUSTOM_CVAR (String, language, "auto", CVAR_ARCHIVE|CVAR_NOINITCALL)
 {
 	SetLanguageIDs ();
-	GStrings.LoadStrings (false);
+	GStrings.UpdateLanguage();
 	for (auto Level : AllLevels())
 	{
 		// does this even make sense on secondary levels...?

src/gamedata/d_dehacked.cpp

@@ -285,7 +285,7 @@ static bool  including, includenotext;
 
 static const char *unknown_str = "Unknown key %s encountered in %s %d.\n";
 
-static FStringTable *EnglishStrings;
+static StringMap EnglishStrings, DehStrings;
 
 // This is an offset to be used for computing the text stuff.
 // Straight from the DeHackEd source which was
@@ -2169,7 +2169,7 @@ static int PatchMusic (int dummy)
 		
 		keystring << "MUSIC_" << Line1;
 
-		GStrings.SetString (keystring, newname);
+		DehStrings.Insert(keystring, newname);
 		DPrintf (DMSG_SPAMMY, "Music %s set to:\n%s\n", keystring.GetChars(), newname);
 	}
 
@@ -2280,11 +2280,11 @@ static int PatchText (int oldSize)
 	const char *str;
 	do
 	{
-		str = EnglishStrings->MatchString(oldStr);
+		str = EnglishStrings.MatchString(oldStr);
 		if (str != NULL)
 		{
-			GStrings.SetString(str, newStr);
+			DehStrings.Insert(str, newStr);
-			EnglishStrings->SetString(str, "~~");	// set to something invalid so that it won't get found again by the next iteration or  by another replacement later
+			EnglishStrings.Remove(str);	// remove entry so that it won't get found again by the next iteration or  by another replacement later
 			good = true;
 		}
 	} 
@@ -2337,7 +2337,7 @@ static int PatchStrings (int dummy)
 		// Account for a discrepancy between Boom's and ZDoom's name for the red skull key pickup message
 		const char *ll = Line1;
 		if (!stricmp(ll, "GOTREDSKULL")) ll = "GOTREDSKUL";
-		GStrings.SetString (ll, holdstring);
+		DehStrings.Insert(ll, holdstring);
 		DPrintf (DMSG_SPAMMY, "%s set to:\n%s\n", Line1, holdstring.GetChars());
 	}
 
@@ -2670,11 +2670,6 @@ static void UnloadDehSupp ()
 		StyleNames.Reset();
 		AmmoNames.Reset();
 		UnchangedSpriteNames.Reset();
-		if (EnglishStrings != NULL)
-		{
-			delete EnglishStrings;
-			EnglishStrings = NULL;
-		}
 	}
 }
 
@@ -2704,12 +2699,8 @@ static bool LoadDehSupp ()
 			return true;
 		}
 
-		if (EnglishStrings == NULL)
+		if (EnglishStrings.CountUsed() == 0)
-		{
+			EnglishStrings = GStrings.GetDefaultStrings();
-			EnglishStrings = new FStringTable;
-			EnglishStrings->LoadStrings (true);
-		}
-
 
 		UnchangedSpriteNames.Resize(sprites.Size());
 		for (unsigned i = 0; i < UnchangedSpriteNames.Size(); ++i)
@@ -3079,6 +3070,8 @@ void FinishDehPatch ()
 	StateMap.ShrinkToFit();
 	TouchedActors.Clear();
 	TouchedActors.ShrinkToFit();
+	EnglishStrings.Clear();
+	GStrings.SetDehackedStrings(std::move(DehStrings));
 
 	// Now it gets nasty: We have to fiddle around with the weapons' ammo use info to make Doom's original
 	// ammo consumption work as intended.

src/gamedata/stringtable.cpp

@@ -43,119 +43,31 @@
 #include "v_text.h"
 #include "gi.h"
 
-// PassNum identifies which language pass this string is from.
-// PassNum 0 is for DeHacked.
-// PassNum 1 is for * strings.
-// PassNum 2+ are for specific locales.
 
-struct FStringTable::StringEntry
+void FStringTable::LoadStrings ()
-{
-	StringEntry *Next;
-	char *Name;
-	uint8_t PassNum;
-	char String[];
-};
-
-FStringTable::FStringTable ()
-{
-	for (int i = 0; i < HASH_SIZE; ++i)
-	{
-		Buckets[i] = NULL;
-	}
-}
-
-FStringTable::~FStringTable ()
-{
-	FreeData ();
-}
-
-void FStringTable::FreeData ()
-{
-	for (int i = 0; i < HASH_SIZE; ++i)
-	{
-		StringEntry *entry = Buckets[i], *next;
-		Buckets[i] = NULL;
-		while (entry != NULL)
-		{
-			next = entry->Next;
-			M_Free (entry);
-			entry = next;
-		}
-	}
-}
-
-void FStringTable::FreeNonDehackedStrings ()
-{
-	for (int i = 0; i < HASH_SIZE; ++i)
-	{
-		StringEntry *entry, *next, **pentry;
-
-		for (pentry = &Buckets[i], entry = *pentry; entry != NULL; )
-		{
-			next = entry->Next;
-			if (entry->PassNum != 0)
-			{
-				*pentry = next;
-				M_Free (entry);
-			}
-			else
-			{
-				pentry = &entry->Next;
-			}
-			entry = next;
-		}
-	}
-}
-
-void FStringTable::LoadStrings (bool enuOnly)
 {
 	int lastlump, lump;
-	int i, j;
-
-	FreeNonDehackedStrings ();
 
 	lastlump = 0;
 
 	while ((lump = Wads.FindLump ("LANGUAGE", &lastlump)) != -1)
 	{
-		j = 0;
+		LoadLanguage (lump);
-		if (!enuOnly)
+	}
+	UpdateLanguage();
+}
+
+void FStringTable::LoadLanguage (int lumpnum)
 {
-			LoadLanguage (lump, MAKE_ID('*',0,0,0), true, ++j);
+	bool errordone = false;
-			for (i = 0; i < 4; ++i)
+	TArray<uint32_t> activeMaps;
-			{
-				LoadLanguage (lump, LanguageIDs[i], true, ++j);
-				LoadLanguage (lump, LanguageIDs[i] & MAKE_ID(0xff,0xff,0,0), true, ++j);
-				LoadLanguage (lump, LanguageIDs[i], false, ++j);
-			}
-		}
-
-		// Fill in any missing strings with the default language
-		LoadLanguage (lump, MAKE_ID('*','*',0,0), true, ++j);
-	}
-}
-
-void FStringTable::LoadLanguage (int lumpnum, uint32_t code, bool exactMatch, int passnum)
-{
-	static bool errordone = false;
-	const uint32_t orMask = exactMatch ? 0 : MAKE_ID(0,0,0xff,0);
-	uint32_t inCode = 0;
-	StringEntry *entry, **pentry;
-	uint32_t bucket;
-	int cmpval;
-	bool skip = true;
-
-	code |= orMask;
-
 	FScanner sc(lumpnum);
 	sc.SetCMode (true);
 	while (sc.GetString ())
 	{
 		if (sc.Compare ("["))
 		{ // Process language identifiers
-			bool donot = false;
+			activeMaps.Clear();
-			bool forceskip = false;
-			skip = true;
 			sc.MustGetString ();
 			do
 			{
@@ -164,17 +76,18 @@ void FStringTable::LoadLanguage (int lumpnum, uint32_t code, bool exactMatch, in
 				{
 					if (len == 1 && sc.String[0] == '~')
 					{
-						donot = true;
+						// deprecated and ignored
+						sc.ScriptMessage("Deprecated option '~' found in language list");
 						sc.MustGetString ();
 						continue;
 					}
 					if (len == 1 && sc.String[0] == '*')
 					{
-						inCode = MAKE_ID('*',0,0,0);
+						activeMaps.Push(MAKE_ID('*', 0, 0, 0));
 					}
 					else if (len == 7 && stricmp (sc.String, "default") == 0)
 					{
-						inCode = MAKE_ID('*','*',0,0);
+						activeMaps.Push(MAKE_ID('*', '*', 0, 0));
 					}
 					else
 					{
@@ -184,31 +97,14 @@ void FStringTable::LoadLanguage (int lumpnum, uint32_t code, bool exactMatch, in
 				}
 				else
 				{
-					inCode = MAKE_ID(tolower(sc.String[0]), tolower(sc.String[1]), tolower(sc.String[2]), 0);
+					activeMaps.Push(MAKE_ID(tolower(sc.String[0]), tolower(sc.String[1]), tolower(sc.String[2]), 0));
-				}
-				if ((inCode | orMask) == code)
-				{
-					if (donot)
-					{
-						forceskip = true;
-						donot = false;
-					}
-					else
-					{
-						skip = false;
-					}
 				}
 				sc.MustGetString ();
 			} while (!sc.Compare ("]"));
-			if (donot)
-			{
-				sc.ScriptError ("You must specify a language after ~");
-			}
-			skip |= forceskip;
 		}
 		else
 		{ // Process string definitions.
-			if (inCode == 0)
+			if (activeMaps.Size() == 0)
 			{
 				// LANGUAGE lump is bad. We need to check if this is an old binary
 				// lump and if so just skip it to allow old WADs to run which contain
@@ -222,7 +118,7 @@ void FStringTable::LoadLanguage (int lumpnum, uint32_t code, bool exactMatch, in
 				sc.ScriptError ("Found a string without a language specified.");
 			}
 
-			bool savedskip = skip;
+			bool skip = false;
 			if (sc.Compare("$"))
 			{
 				sc.MustGetStringName("ifgame");
@@ -234,20 +130,7 @@ void FStringTable::LoadLanguage (int lumpnum, uint32_t code, bool exactMatch, in
 
 			}
 
-			if (skip)
+			FName strName (sc.String);
-			{ // We're not interested in this language, so skip the string.
-				sc.MustGetStringName ("=");
-				sc.MustGetString ();
-				do
-				{
-					sc.MustGetString ();
-				} 
-				while (!sc.Compare (";"));
-				skip = savedskip;
-				continue;
-			}
-
-			FString strName (sc.String);
 			sc.MustGetStringName ("=");
 			sc.MustGetString ();
 			FString strText (sc.String, ProcessEscapes (sc.String));
@@ -258,37 +141,34 @@ void FStringTable::LoadLanguage (int lumpnum, uint32_t code, bool exactMatch, in
 				strText += sc.String;
 				sc.MustGetString ();
 			}
+			// Insert the string into all relevant tables.
+			for (auto map : activeMaps)
+			{
+				allStrings[map].Insert(strName, strText);
+			}
+		}
+	}
+}
 
-			// Does this string exist? If so, should we overwrite it?
+void FStringTable::UpdateLanguage()
-			bucket = MakeKey (strName.GetChars()) & (HASH_SIZE-1);
-			pentry = &Buckets[bucket];
-			entry = *pentry;
-			cmpval = 1;
-			while (entry != NULL)
 {
-				cmpval = stricmp (entry->Name, strName.GetChars());
+	currentLanguageSet.Clear();
-				if (cmpval >= 0)
+
-					break;
+	auto checkone = [&](uint32_t lang_id)
-				pentry = &entry->Next;
-				entry = *pentry;
-			}
-			if (cmpval == 0 && entry->PassNum >= passnum)
 	{
-				*pentry = entry->Next;
+		auto list = allStrings.CheckKey(lang_id);
-				M_Free (entry);
+		if (list && currentLanguageSet.Find(list) == currentLanguageSet.Size())
-				entry = NULL;
+			currentLanguageSet.Push(list);
-			}
+	};
-			if (entry == NULL || cmpval > 0)
+
+	checkone(MAKE_ID('*', '*', '*', 0));
+	checkone(MAKE_ID('*', 0, 0, 0));
+	for (int i = 0; i < 4; ++i)
 	{
-				entry = (StringEntry *)M_Malloc (sizeof(*entry) + strText.Len() + strName.Len() + 2);
+		checkone(LanguageIDs[i]);
-				entry->Next = *pentry;
+		checkone(LanguageIDs[i] & MAKE_ID(0xff, 0xff, 0, 0));
-				*pentry = entry;
-				strcpy (entry->String, strText.GetChars());
-				strcpy (entry->Name = entry->String + strText.Len() + 1, strName.GetChars());
-				entry->PassNum = passnum;
-			}
-		}
 	}
+	checkone(MAKE_ID('*', '*', 0, 0));
 }
 
 // Replace \ escape sequences in a string with the escaped characters.
@@ -321,27 +201,20 @@ size_t FStringTable::ProcessEscapes (char *iptr)
 // Finds a string by name and returns its value
 const char *FStringTable::operator[] (const char *name) const
 {
-	if (name == NULL)
+	if (name == nullptr || *name == 0)
 	{
-		return NULL;
+		return nullptr;
 	}
-	uint32_t bucket = MakeKey (name) & (HASH_SIZE - 1);
+	FName nm(name, true);
-	StringEntry *entry = Buckets[bucket];
+	if (nm != NAME_None)
-
-	while (entry != NULL)
 	{
-		int cmpval = stricmp (entry->Name, name);
+		for (auto map : currentLanguageSet)
-		if (cmpval == 0)
 		{
-			return entry->String;
+			auto item = map->CheckKey(nm);
+			if (item) return item->GetChars();
 		}
-		if (cmpval == 1)
-		{
-			return NULL;
 	}
-		entry = entry->Next;
+	return nullptr;
-	}
-	return NULL;
 }
 
 // Finds a string by name and returns its value. If the string does
@@ -352,75 +225,19 @@ const char *FStringTable::operator() (const char *name) const
 	return str ? str : name;
 }
 
-// Find a string by name. pentry1 is a pointer to a pointer to it, and entry1 is a
-// pointer to it. Return NULL for entry1 if it wasn't found.
-void FStringTable::FindString (const char *name, StringEntry **&pentry1, StringEntry *&entry1)
-{
-	uint32_t bucket = MakeKey (name) & (HASH_SIZE - 1);
-	StringEntry **pentry = &Buckets[bucket], *entry = *pentry;
-
-	while (entry != NULL)
-	{
-		int cmpval = stricmp (entry->Name, name);
-		if (cmpval == 0)
-		{
-			pentry1 = pentry;
-			entry1 = entry;
-			return;
-		}
-		if (cmpval == 1)
-		{
-			pentry1 = pentry;
-			entry1 = NULL;
-			return;
-		}
-		pentry = &entry->Next;
-		entry = *pentry;
-	}
-	pentry1 = pentry;
-	entry1 = entry;
-}
 
 // Find a string with the same exact text. Returns its name.
-const char *FStringTable::MatchString (const char *string) const
+const char *StringMap::MatchString (const char *string) const
 {
-	for (int i = 0; i < HASH_SIZE; ++i)
+	StringMap::ConstIterator it(*this);
-	{
+	StringMap::ConstPair *pair;
-		for (StringEntry *entry = Buckets[i]; entry != NULL; entry = entry->Next)
-		{
-			if (strcmp (entry->String, string) == 0)
-			{
-				return entry->Name;
-			}
-		}
-	}
-	return NULL;
-}
 
-void FStringTable::SetString (const char *name, const char *newString)
+	while (it.NextPair(pair))
 	{
-	StringEntry **pentry, *oentry;
+		if (pair->Value.Compare(string) == 0)
-	FindString (name, pentry, oentry);
-
-	size_t newlen = strlen (newString);
-	size_t namelen = strlen (name);
-
-	// Create a new string entry
-	StringEntry *entry = (StringEntry *)M_Malloc (sizeof(*entry) + newlen + namelen + 2);
-	strcpy (entry->String, newString);
-	strcpy (entry->Name = entry->String + newlen + 1, name);
-	entry->PassNum = 0;
-
-	// If this is a new string, insert it. Otherwise, replace the old one.
-	if (oentry == NULL)
 		{
-		entry->Next = *pentry;
+			return pair->Key.GetChars();
-		*pentry = entry;
-	}
-	else
-	{
-		*pentry = entry;
-		entry->Next = oentry->Next;
-		M_Free (oentry);
 		}
 	}
+	return nullptr;
+}

src/gamedata/stringtable.h

@@ -44,34 +44,41 @@
 
 
 #include <stdlib.h>
+#include "doomdef.h"
 #include "doomtype.h"
 
+// This public interface is for Dehacked
+class StringMap : public TMap<FName, FString>
+{
+public:
+	const char *MatchString(const char *string) const;
+};
+
+
 class FStringTable
 {
 public:
-	struct StringEntry;
+	using LangMap = TMap<uint32_t, StringMap>;
 
-	FStringTable ();
+	void LoadStrings ();
-	~FStringTable ();
+	void UpdateLanguage();
-
+	StringMap GetDefaultStrings() { return allStrings[MAKE_ID('*', '*', 0, 0)]; }	// Dehacked needs these for comparison
-	void LoadStrings (bool enuOnly);
+	void SetDehackedStrings(StringMap && map)
+	{
+		allStrings.Insert(MAKE_ID('*', '*', '*', 0), map);
+		UpdateLanguage();
+	}
 
 	const char *operator() (const char *name) const;	// Never returns NULL
 	const char *operator[] (const char *name) const;	// Can return NULL
 
-	const char *MatchString (const char *string) const;
-	void SetString (const char *name, const char *newString);
-
 private:
-	enum { HASH_SIZE = 128 };
 
-	StringEntry *Buckets[HASH_SIZE];
+	LangMap allStrings;
+	TArray<StringMap*> currentLanguageSet;
 
-	void FreeData ();
+	void LoadLanguage (int lumpnum);
-	void FreeNonDehackedStrings ();
-	void LoadLanguage (int lumpnum, uint32_t code, bool exactMatch, int passnum);
 	static size_t ProcessEscapes (char *str);
-	void FindString (const char *stringName, StringEntry **&pentry, StringEntry *&entry);
 };
 
 #endif //__STRINGTABLE_H__

wadsrc/static/language.enu

@@ -1,6 +1,6 @@
 /* U.S. English. (Sorry, it's not English English.) */
 
-[enu default]
+[en default]
 
 SECRETMESSAGE = "A secret is revealed!";