rewrote FileSystem::AddFromBuffer so that it gets backed by an actual FResourceFile.

source/CMakeLists.txt

@@ -593,6 +593,7 @@ file( GLOB HEADER_FILES
 	common/thirdparty/rapidjson/*.h
 	common/thirdparty/math/*h
 	common/thirdparty/libsmackerdec/include/*.h
+	common/thirdparty/utf8proc/*.h
 	common/rendering/*.h
 	common/rendering/gl_load/*.h
 	common/rendering/hwrenderer/*.h
@@ -1244,6 +1245,7 @@ set( GAME_SOURCES
 	${PCH_SOURCES}
 	common/utility/x86.cpp
 	common/thirdparty/strnatcmp.c
+	common/thirdparty/utf8proc/utf8proc.c
 	common/thirdparty/stb/stb_sprintf.c
 	common/utility/zstring.cpp
 	common/utility/findfile.cpp
@@ -1283,6 +1285,7 @@ set( GAME_SOURCES
 	common/filesystem/source/files_decompress.cpp
 	common/filesystem/source/fs_findfile.cpp
 	common/filesystem/source/fs_stringpool.cpp
+	common/filesystem/source/unicode.cpp
 
 )
 
@@ -1341,6 +1344,7 @@ include_directories(
 	common/thirdparty/libsmackerdec/include
 	common/thirdparty
 	common/thirdparty/stb
+	common/thirdparty/utf8proc
 	common/textures
 	common/textures/formats
 	common/textures/hires
@@ -1553,6 +1557,7 @@ source_group("Common\\Third Party\\Math" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SO
 source_group("Common\\Third Party\\RapidJSON" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/rapidjson/.+")
 source_group("Common\\Third Party\\SFMT" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/sfmt/.+")
 source_group("Common\\Third Party\\stb" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/stb/.+")
+source_group("Common\\Third Party\\utf8proc" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/utf8proc/.+")
 source_group("Utility\\Smackerdec" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/smackerdec/.+")
 source_group("Utility\\Smackerdec\\Headers" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/libsmackerdec/include/.+")
 source_group("Utility\\Smackerdec\\Sources" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/libsmackerdec/src/.+")

source/common/engine/serializer.cpp

@@ -762,9 +762,9 @@ FCompressedBuffer FSerializer::GetCompressedOutput()
 	int err;
 
 	stream.next_in = (Bytef *)w->mOutString.GetString();
-	stream.avail_in = buff.mSize;
+	stream.avail_in = (unsigned)buff.mSize;
 	stream.next_out = (Bytef*)compressbuf;
-	stream.avail_out = buff.mSize;
+	stream.avail_out = (unsigned)buff.mSize;
 	stream.zalloc = (alloc_func)0;
 	stream.zfree = (free_func)0;
 	stream.opaque = (voidpf)0;

source/common/filesystem/include/fs_filesystem.h

@@ -169,9 +169,7 @@ public:
 		return (int)Files.size();
 	}
 
-	void AddLump(FResourceLump* lump);
+	int AddFromBuffer(const char* name, char* data, int size, int id, int flags);
-	int AddExternalFile(const char *filename);
-	int AddFromBuffer(const char* name, const char* type, char* data, int size, int id, int flags);
 	FileReader* GetFileReader(int wadnum);	// Gets a FileReader object to the entire WAD
 	void InitHashChains();
 

source/common/filesystem/source/filesystem.cpp

@@ -287,35 +287,6 @@ bool FileSystem::InitMultipleFiles (std::vector<std::string>& filenames, LumpFil
 	return true;
 }
 
-//==========================================================================
-//
-// AddLump
-//
-// Adds a given lump to the directory. Does not perform rehashing
-//
-//==========================================================================
-
-void FileSystem::AddLump(FResourceLump *lump)
-{
-	FileInfo.resize(FileInfo.size() + 1);
-	FileInfo.back().SetFromLump(-1, lump, stringpool);
-}
-
-//-----------------------------------------------------------------------
-//
-// Adds an external file to the lump list but not to the hash chains
-// It's just a simple means to assign a lump number to some file so that
-// the texture manager can read from it.
-//
-//-----------------------------------------------------------------------
-
-int FileSystem::AddExternalFile(const char *filename)
-{
-	FResourceLump *lump = new FExternalLump(filename, -1, stringpool);
-	AddLump(lump);
-	return (int)FileInfo.size() - 1;	// later
-}
-
 //==========================================================================
 //
 // AddFromBuffer
@@ -324,27 +295,24 @@ int FileSystem::AddExternalFile(const char *filename)
 //
 //==========================================================================
 
-struct FMemoryLump : public FResourceLump
+FResourceFile* CheckLump(const char* filename, FileReader& file, LumpFilterInfo*, FileSystemMessageFunc Printf, StringPool* sp);
-{
-	FMemoryLump(const void* data, int length)
-	{
-		RefCount = -1;
-		LumpSize = length;
-		Cache = new char[length];
-		memcpy(Cache, data, length);
-	}
-};
 
-int FileSystem::AddFromBuffer(const char* name, const char* type, char* data, int size, int id, int flags)
+int FileSystem::AddFromBuffer(const char* name, char* data, int size, int id, int flags)
 {
-	std::string fullname = name;
+	FileReader fr;
-	fullname += '.';
+	fr.OpenMemoryArray((uint8_t*)data, size);
-	fullname += type;
+
-	auto newlump = new FMemoryLump(data, size);
+	// just wrap this into a single lump resource file (should be done a little better later.)
-	newlump->LumpNameSetup(fullname.c_str(), stringpool);
+	auto rf = CheckLump(name, fr, nullptr, nullptr, stringpool);
-	AddLump(newlump);
+	if (rf)
-	FileInfo.back().resourceId = id;
+	{
-	return (int)FileInfo.size()-1;
+		Files.push_back(rf);
+		FResourceLump* lump = rf->GetLump(0);
+		FileInfo.resize(FileInfo.size() + 1);
+		FileSystem::LumpRecord* lump_p = &FileInfo.back();
+		lump_p->SetFromLump((int)Files.size(), lump, stringpool);
+	}
+	return (int)FileInfo.size() - 1;
 }
 
 //==========================================================================

source/common/fonts/hexfont.cpp

@@ -63,7 +63,7 @@ struct HexDataSource
 		FScanner sc;
 
 		auto data = resf->Read(index);
-		sc.OpenMem("newconsolefont.hex", data.string(), data.size());
+		sc.OpenMem("newconsolefont.hex", data.string(), (int)data.size());
 		sc.SetCMode(true);
 		glyphdata.Push(0);	// ensure that index 0 can be used as 'not present'.
 		while (sc.GetString())

source/common/models/models_iqm.cpp

@@ -137,7 +137,7 @@ bool IQMModel::Load(const char* path, int lumpnum, const char* buffer, int lengt
 		}
 
 		reader.SeekTo(ofs_anims);
-		for(int i = 0; i < Anims.Size(); i++)
+		for(unsigned i = 0; i < Anims.Size(); i++)
 		{
 			IQMAnim& anim = Anims[i];
 			anim.Name = reader.ReadName(text);

source/common/scripting/core/types.cpp

@@ -2568,7 +2568,7 @@ static void PMapValueWriter(FSerializer &ar, const M *map, const PMap *m)
 					{ // invalid
 						m->ValueType->WriteValue(ar,"invalid",static_cast<const void *>(&p->Value));
 					}
-					else if(p->Key == 0 || p->Key >= TexMan.NumTextures())
+					else if(p->Key == 0 || p->Key >= (unsigned)TexMan.NumTextures())
 					{ // null
 						m->ValueType->WriteValue(ar,"null",static_cast<const void *>(&p->Value));
 					}
@@ -2693,7 +2693,7 @@ static bool PMapValueReader(FSerializer &ar, M *map, const PMap *m)
 						ar.EndObject();
 						return false;
 					}
-					int v = s.ToULong();
+					int v = (int)s.ToULong();
 
 					if (sysCallbacks.RemapTranslation) v = sysCallbacks.RemapTranslation(FTranslationID::fromInt(v)).index();
 

source/core/rts.cpp

@@ -119,10 +119,12 @@ bool RTS_IsInitialized()
 	{
 		if (li.size > 0)
 		{
-			FStringf rts("rts%02d", i);
+			char rts[16];
-			int lump = fileSystem.AddFromBuffer(rts.GetChars(), "rts", (char*)RTSFile.Data() + li.position, li.size, -1, 0);
+			snprintf(rts, 16, "rts%02d.rts", i);
-			li.sid = soundEngine->AddSoundLump(rts.GetChars(), lump, 0, -1);
+			int lump = fileSystem.AddFromBuffer(rts, (char*)RTSFile.Data() + li.position, li.size, -1, 0);
+			li.sid = soundEngine->AddSoundLump(rts, lump, 0, -1);
 		}
+		i++;
 	}
 	return false;
 }

source/games/blood/src/barf.cpp

@@ -941,7 +941,7 @@ void addMemoryResource(const char* filePath, int flags, int ID)
 {
 	FString zDirectory, zFilename, zType;
 	SplitPath(filePath, zDirectory, zFilename, zType, true);
-	fileSystem.AddFromBuffer(zFilename.GetChars(), zType.GetChars(), buffer, nBytes, ID, flags);
+	fileSystem.AddFromBuffer((zFilename + "." + zType).GetChars(), buffer, nBytes, ID, flags);
 }