filesystem update from GZDoom.

source/CMakeLists.txt

@@ -1122,6 +1122,7 @@ set (PCH_SOURCES
 	common/utility/s_playlist.cpp
 	common/utility/name.cpp
 	common/utility/r_memory.cpp
+	common/utility/writezip.cpp
 	common/thirdparty/base64.cpp
 	common/thirdparty/md5.cpp
  	common/thirdparty/superfasthash.cpp
@@ -1279,6 +1280,8 @@ set( GAME_SOURCES
 	common/filesystem/source/file_pak.cpp
 	common/filesystem/source/file_whres.cpp
 	common/filesystem/source/file_ssi.cpp
+	common/filesystem/source/file_hog.cpp
+	common/filesystem/source/file_mvl.cpp
 	common/filesystem/source/file_directory.cpp
 	common/filesystem/source/resourcefile.cpp
 	common/filesystem/source/files.cpp
@@ -1286,6 +1289,7 @@ set( GAME_SOURCES
 	common/filesystem/source/fs_findfile.cpp
 	common/filesystem/source/fs_stringpool.cpp
 	common/filesystem/source/unicode.cpp
+	common/filesystem/source/critsec.cpp
 
 )
 

source/common/audio/music/i_music.cpp

@@ -182,8 +182,8 @@ static void SetupGenMidi()
 	}
 	auto genmidi = fileSystem.ReadFile(lump);
 
-	if (genmidi.GetSize() < 8 + 175 * 36 || memcmp(genmidi.GetMem(), "#OPL_II#", 8)) return;
+	if (genmidi.size() < 8 + 175 * 36 || memcmp(genmidi.data(), "#OPL_II#", 8)) return;
-	ZMusic_SetGenMidi(genmidi.GetBytes() + 8);
+	ZMusic_SetGenMidi(genmidi.bytes() + 8);
 }
 
 static void SetupWgOpn()
@@ -194,7 +194,7 @@ static void SetupWgOpn()
 		return;
 	}
 	auto data = fileSystem.ReadFile(lump);
-	ZMusic_SetWgOpn(data.GetMem(), (uint32_t)data.GetSize());
+	ZMusic_SetWgOpn(data.data(), (uint32_t)data.size());
 }
 
 static void SetupDMXGUS()
@@ -206,7 +206,7 @@ static void SetupDMXGUS()
 		return;
 	}
 	auto data = fileSystem.ReadFile(lump);
-	ZMusic_SetDmxGus(data.GetMem(), (uint32_t)data.GetSize());
+	ZMusic_SetDmxGus(data.data(), (uint32_t)data.size());
 }
 
 #endif

source/common/audio/music/i_soundfont.cpp

@@ -218,7 +218,7 @@ FileReader FZipPatReader::OpenFile(const char *name)
 		auto lump = resf->FindEntry(name);
 		if (lump >= 0)
 		{
-			return resf->GetEntryReader(lump);
+			return resf->GetEntryReader(lump, FileSys::READER_NEW, FileSys::READERFLAG_SEEKABLE);
 		}
 	}
 	fr.OpenFile(name);

source/common/audio/music/music.cpp

@@ -703,6 +703,8 @@ bool S_ChangeMusic(const char* musicname, int order, bool looping, bool force)
 	// opening the music must be done by the game because it's different depending on the game's file system use.
 	FileReader reader = mus_cb.OpenMusic(musicname);
 	if (!reader.isOpen()) return false;
+	auto m = reader.Read();
+	reader.Seek(0, FileReader::SeekSet);
 
 	// shutdown old music
 	S_StopMusic(true);

source/common/console/c_cvars.cpp

@@ -44,6 +44,7 @@
 #include "printf.h"
 #include "palutil.h"
 #include "i_interface.h"
+#include "gstrings.h"
 
 #include "dobject.h"
 #include "dobjtype.h"
@@ -1695,16 +1696,37 @@ CCMD (toggle)
 	}
 }
 
-void FBaseCVar::ListVars (const char *filter, bool plain)
+void FBaseCVar::ListVars (const char *filter, int listtype)
 {
 	int count = 0;
 
+	bool plain = listtype == LCT_Plain;
+	bool includedesc = listtype == LCT_FullSearch;
+
 	decltype(cvarMap)::Iterator it(cvarMap);
 	decltype(cvarMap)::Pair *pair;
 	while (it.NextPair(pair))
 	{
 		auto var = pair->Value;
-		if (CheckWildcards (filter, var->GetName()))
+
+		bool ismatch;
+
+		if (filter && includedesc)
+		{
+			// search always allow partial matches
+			// also allow matching to cvar name, localised description, and description language-id
+
+			FString SearchString = FString("*") + filter + "*";
+			ismatch = CheckWildcards (SearchString.GetChars(), var->GetName()) ||
+				CheckWildcards (SearchString.GetChars(), var->GetDescription().GetChars()) ||
+				CheckWildcards (SearchString.GetChars(), GStrings.localize(var->GetDescription().GetChars()));
+		}
+		else
+		{
+			ismatch = CheckWildcards (filter, var->GetName());
+		}
+
+		if (ismatch)
 		{
 			uint32_t flags = var->GetFlags();
 			if (plain)
@@ -1718,7 +1740,8 @@ void FBaseCVar::ListVars (const char *filter, bool plain)
 			else
 			{
 				++count;
-				Printf ("%c%c%c%c%c %s = %s\n",
+
+				Printf ("%c%c%c%c%c %s = %s",
 					flags & CVAR_ARCHIVE ? 'A' : ' ',
 					flags & CVAR_USERINFO ? 'U' :
 						flags & CVAR_SERVERINFO ? 'S' :
@@ -1730,6 +1753,16 @@ void FBaseCVar::ListVars (const char *filter, bool plain)
 					flags & CVAR_IGNORE ? 'X' : ' ',
 					var->GetName(),
 					var->GetHumanString());
+
+				if (includedesc)
+					if (var->GetDescription().Len())
+						Printf(" // \"%s\"\n", GStrings.localize(var->GetDescription().GetChars()));
+					else
+						Printf("\n");
+				else
+					Printf("\n");
+				
+
 			}
 		}
 	}
@@ -1740,17 +1773,29 @@ CCMD (cvarlist)
 {
 	if (argv.argc() == 1)
 	{
-		FBaseCVar::ListVars (NULL, false);
+		FBaseCVar::ListVars (NULL, LCT_Default);
 	}
 	else
 	{
-		FBaseCVar::ListVars (argv[1], false);
+		FBaseCVar::ListVars (argv[1], LCT_Default);
 	}
 }
 
 CCMD (cvarlistplain)
 {
-	FBaseCVar::ListVars (NULL, true);
+	FBaseCVar::ListVars (NULL, LCT_Plain);
+}
+
+CCMD (cvarsearch)
+{
+	if (argv.argc() == 1)
+	{
+		FBaseCVar::ListVars (NULL, LCT_FullSearch);
+	}
+	else
+	{
+		FBaseCVar::ListVars (argv[1], LCT_FullSearch);
+	}
 }
 
 CCMD (archivecvar)

source/common/console/c_cvars.h

@@ -89,6 +89,13 @@ enum ECVarType
 	CVAR_Dummy,			// Unknown
 };
 
+enum ListCCMDType
+{
+	LCT_Default,
+	LCT_Plain,
+	LCT_FullSearch,
+};
+
 
 class FIntCVarRef;
 union UCVarValue
@@ -201,7 +208,7 @@ public:
 	static void MarkZSCallbacks ();
 	static void ResetColors ();		// recalc color cvars' indices after screen change
 
-	static void ListVars (const char *filter, bool plain);
+	static void ListVars (const char *filter, int listtype);
 
 	const FString &GetDescription() const { return Description; };
 	const FString& GetToggleMessage(int which) { return ToggleMessages[which]; }

source/common/cutscenes/movieplayer.cpp

@@ -153,7 +153,7 @@ class AnmPlayer : public MoviePlayer
 {
 	// This doesn't need its own class type
 	anim_t anim;
-	FileSys::ResourceData buffer;
+	FileSys::FileData buffer;
 	int numframes = 0;
 	int curframe = 1;
 	int frametime = 0;
@@ -515,7 +515,7 @@ public:
 		}
 		else if (soundtrack >= 0)
 		{
-			FileReader reader = fileSystem.OpenFileReader(soundtrack);
+			FileReader reader = fileSystem.ReopenFileReader(soundtrack);
 			if (reader.isOpen())
 			{
 				MusicStream = ZMusic_OpenSong(GetMusicReader(reader), MDEV_DEFAULT, nullptr);
@@ -794,7 +794,7 @@ public:
 					if (sound == INVALID_SOUND)
 						soundEngine->StopAllChannels();
 					else
-						soundEngine->StartSound(SOURCE_None, nullptr, nullptr, CHAN_AUTO, nostopsound ? CHANF_UI | CHANF_FORCE : CHANF_FORCE, sound, 1.f, ATTN_NONE);
+						soundEngine->StartSound(SOURCE_None, nullptr, nullptr, CHAN_AUTO, nostopsound ? CHANF_UI : CHANF_NONE, sound, 1.f, ATTN_NONE);
 				}
 			}
 		}
@@ -838,10 +838,10 @@ MoviePlayer* OpenMovie(const char* filename, TArray<int>& ans, const int* framet
 	{
 		auto fn = StripExtension(filename);
 		DefaultExtension(fn, ".ivf");
-		fr = fileSystem.OpenFileReader(fn.GetChars());
+		fr = fileSystem.ReopenFileReader(fn.GetChars());
 	}
 
-	if (!fr.isOpen()) fr = fileSystem.OpenFileReader(filename);
+	if (!fr.isOpen()) fr = fileSystem.ReopenFileReader(filename);
 	if (!fr.isOpen())
 	{
 		size_t nLen = strlen(filename);
@@ -849,7 +849,7 @@ MoviePlayer* OpenMovie(const char* filename, TArray<int>& ans, const int* framet
 		if (nLen >= 3 && isalpha(filename[0]) && filename[1] == ':' && filename[2] == '/')
 		{
 			filename += 3;
-			fr = fileSystem.OpenFileReader(filename);
+			fr = fileSystem.ReopenFileReader(filename);
 		}
 		if (!fr.isOpen())
 		{

source/common/engine/sc_man.cpp

@@ -200,10 +200,10 @@ void FScanner :: OpenLumpNum (int lump)
 {
 	Close ();
 	{
-		auto mem = fileSystem.OpenFileReader(lump);
+		auto len = fileSystem.FileLength(lump);
-		auto buff = ScriptBuffer.LockNewBuffer(mem.GetLength());
+		auto buff = ScriptBuffer.LockNewBuffer(len);
-		mem.Read(buff, mem.GetLength());
+		fileSystem.ReadFile(lump, buff);
-		buff[mem.GetLength()] = 0;
+		buff[len] = 0;
 		ScriptBuffer.UnlockBuffer();
 	}
 	ScriptName = fileSystem.GetFileFullPath(lump).c_str();

source/common/engine/serializer.cpp

@@ -753,7 +753,6 @@ FCompressedBuffer FSerializer::GetCompressedOutput()
 	EndObject();
 	buff.filename = nullptr;
 	buff.mSize = (unsigned)w->mOutString.GetSize();
-	buff.mZipFlags = 0;
 	buff.mCRC32 = crc32(0, (const Bytef*)w->mOutString.GetString(), buff.mSize);
 
 	uint8_t *compressbuf = new uint8_t[buff.mSize+1];

source/common/engine/stringtable.cpp

@@ -63,8 +63,8 @@ void FStringTable::LoadStrings (const char *language)
 	{
 		auto lumpdata = fileSystem.ReadFile(lump);
 
-		if (!ParseLanguageCSV(lump, lumpdata.GetString(), lumpdata.GetSize()))
+		if (!ParseLanguageCSV(lump, lumpdata.string(), lumpdata.size()))
- 			LoadLanguage (lump, lumpdata.GetString(), lumpdata.GetSize());
+ 			LoadLanguage (lump, lumpdata.string(), lumpdata.size());
 	}
 	UpdateLanguage(language);
 	allMacros.Clear();
@@ -160,7 +160,7 @@ TArray<TArray<FString>> FStringTable::parseCSV(const char* buffer, size_t size)
 bool FStringTable::readMacros(int lumpnum)
 {
 	auto lumpdata = fileSystem.ReadFile(lumpnum);
-	auto data = parseCSV(lumpdata.GetString(), lumpdata.GetSize());
+	auto data = parseCSV(lumpdata.string(), lumpdata.size());
 
 	for (unsigned i = 1; i < data.Size(); i++)
 	{

source/common/filesystem/include/fs_decompress.h

@@ -0,0 +1,63 @@
+#pragma once
+#include "fs_files.h"
+
+namespace FileSys {
+
+// Zip compression methods, extended by some internal types to be passed to OpenDecompressor
+enum ECompressionMethod
+{
+	METHOD_STORED = 0,
+	METHOD_SHRINK = 1,
+	METHOD_IMPLODE = 6,
+	METHOD_DEFLATE = 8,
+	METHOD_BZIP2 = 12,
+	METHOD_LZMA = 14,
+	METHOD_XZ = 95,
+	METHOD_PPMD = 98,
+	METHOD_LZSS = 1337,			// not used in Zips - this is for Console Doom compression
+	METHOD_ZLIB = 1338,			// Zlib stream with header, used by compressed nodes.
+	METHOD_RFFCRYPT = 1339,		// not actual compression but can be put in here to make handling easier.
+	METHOD_IMPLODE_MIN = 1000, // having discrete types for these avoids keeping around the GPFlags word in Zips.
+	METHOD_IMPLODE_0 = 1000,
+	METHOD_IMPLODE_2 = 1002,
+	METHOD_IMPLODE_4 = 1004,
+	METHOD_IMPLODE_6 = 1006,
+	METHOD_IMPLODE_MAX = 1006,
+	METHOD_INVALID = 0x7fff,
+	METHOD_TRANSFEROWNER = 0x8000,
+};
+
+enum EDecompressFlags
+{
+	DCF_TRANSFEROWNER = 1,
+	DCF_SEEKABLE = 2,
+	DCF_EXCEPTIONS = 4,
+	DCF_CACHED = 8,
+};
+
+bool OpenDecompressor(FileReader& self, FileReader &parent, FileReader::Size length, int method, int flags = 0);	// creates a decompressor stream. 'seekable' uses a buffered version so that the Seek and Tell methods can be used.
+
+// This holds a compresed Zip entry with all needed info to decompress it.
+struct FCompressedBuffer
+{
+	size_t mSize;
+	size_t mCompressedSize;
+	int mMethod;
+	unsigned mCRC32;
+	char* mBuffer;
+	const char* filename;
+
+	bool Decompress(char* destbuffer);
+	void Clean()
+	{
+		mSize = mCompressedSize = 0;
+		if (mBuffer != nullptr)
+		{
+			delete[] mBuffer;
+			mBuffer = nullptr;
+		}
+	}
+};
+
+
+}

source/common/filesystem/include/fs_files.h

@@ -40,12 +40,11 @@
 #include <stdio.h>
 #include <stdint.h>
 #include <stdarg.h>
+#include <string.h>
 #include <functional>
 #include <vector>
 #include "fs_swap.h"
 
-#include "tarray.h"
-
 namespace FileSys {
 	
 class FileSystemException : public std::exception
@@ -72,26 +71,10 @@ public:
 	}
 };
 
-// Zip compression methods, extended by some internal types to be passed to OpenDecompressor
-enum
-{
-	METHOD_STORED = 0,
-	METHOD_SHRINK = 1,
-	METHOD_IMPLODE = 6,
-	METHOD_DEFLATE = 8,
-	METHOD_BZIP2 = 12,
-	METHOD_LZMA = 14,
-	METHOD_XZ = 95,
-	METHOD_PPMD = 98,
-	METHOD_LZSS = 1337,	// not used in Zips - this is for Console Doom compression
-	METHOD_ZLIB = 1338,	// Zlib stream with header, used by compressed nodes.
-	METHOD_TRANSFEROWNER = 0x8000,
-};
-
 class FileReader;
 
 // an opaque memory buffer to the file's content. Can either own the memory or just point to an external buffer.
-class ResourceData
+class FileData
 {
 	void* memory;
 	size_t length;
@@ -99,13 +82,29 @@ class ResourceData
 
 public:
 	using value_type = uint8_t;
-	ResourceData() { memory = nullptr; length = 0; owned = true; }
+	FileData() { memory = nullptr; length = 0; owned = true; }
+	FileData(const void* memory_, size_t len, bool own = true)
+	{
+		length = len;
+		if (own)
+		{
+			length = len;
+			memory = allocate(len);
+			if (memory_) memcpy(memory, memory_, len);
+		}
+		else
+		{
+			memory = (void*)memory_;
+			owned = false;
+		}
+	}
+	uint8_t* writable() const { return owned? (uint8_t*)memory : nullptr; }
 	const void* data() const { return memory; }
 	size_t size() const { return length; }
 	const char* string() const { return (const char*)memory; }
 	const uint8_t* bytes() const { return (const uint8_t*)memory; }
 
-	ResourceData& operator = (const ResourceData& copy)
+	FileData& operator = (const FileData& copy)
 	{
 		if (owned && memory) free(memory);
 		length = copy.length;
@@ -119,7 +118,7 @@ public:
 		return *this;
 	}
 
-	ResourceData& operator = (ResourceData&& copy) noexcept
+	FileData& operator = (FileData&& copy) noexcept
 	{
 		if (owned && memory) free(memory);
 		length = copy.length;
@@ -131,13 +130,13 @@ public:
 		return *this;
 	}
 
-	ResourceData(const ResourceData& copy)
+	FileData(const FileData& copy)
 	{
 		memory = nullptr;
 		*this = copy;
 	}
 
-	~ResourceData()
+	~FileData()
 	{
 		if (owned && memory) free(memory);
 	}
@@ -168,6 +167,7 @@ public:
 
 };
 
+
 class FileReaderInterface
 {
 public:
@@ -181,12 +181,8 @@ public:
 	ptrdiff_t GetLength () const { return Length; }
 };
 
-struct FResourceLump;
-
 class FileReader
 {
-	friend struct FResourceLump;	// needs access to the private constructor.
-
 	FileReaderInterface *mReader = nullptr;
 
 	FileReader(const FileReader &r) = delete;
@@ -210,13 +206,13 @@ public:
 
 	FileReader() {}
 
-	FileReader(FileReader &&r)
+	FileReader(FileReader &&r) noexcept
 	{
 		mReader = r.mReader;
 		r.mReader = nullptr;
 	}
 
-	FileReader& operator =(FileReader &&r)
+	FileReader& operator =(FileReader &&r) noexcept
 	{
 		Close();
 		mReader = r.mReader;
@@ -252,11 +248,7 @@ public:
 	bool OpenFile(const char *filename, Size start = 0, Size length = -1, bool buffered = false);
 	bool OpenFilePart(FileReader &parent, Size start, Size length);
 	bool OpenMemory(const void *mem, Size length);	// read directly from the buffer
-	bool OpenMemoryArray(const void *mem, Size length);	// read from a copy of the buffer.
+	bool OpenMemoryArray(FileData& data);	// take the given array
-	bool OpenMemoryArray(std::vector<uint8_t>& data);	// take the given array
-	bool OpenMemoryArray(ResourceData& data);	// take the given array
-	bool OpenMemoryArray(std::function<bool(std::vector<uint8_t>&)> getter);	// read contents to a buffer and return a reader to it
-	bool OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions = false);	// creates a decompressor stream. 'seekable' uses a buffered version so that the Seek and Tell methods can be used.
 
 	Size Tell() const
 	{
@@ -273,36 +265,14 @@ public:
 		return mReader->Read(buffer, len);
 	}
 
-	ResourceData Read(size_t len)
+	FileData Read(size_t len);
-	{
+	FileData ReadPadded(size_t padding);
-		ResourceData buffer;
-		if (len > 0)
-		{
-			Size length = mReader->Read(buffer.allocate(len), len);
-			if ((size_t)length < len) buffer.allocate(length);
-		}
-		return buffer;
-	}
 
-	ResourceData Read()
+	FileData Read()
 	{
 		return Read(GetLength());
 	}
 
-	ResourceData ReadPadded(size_t padding)
-	{
-		auto len = GetLength();
-		ResourceData buffer;
-
-		if (len > 0)
-		{
-			auto p = (char*)buffer.allocate(len + padding);
-			Size length = mReader->Read(p, len);
-			if (length < len) buffer.clear();
-			else memset(p + len, 0, padding);
-		}
-		return buffer;
-	}
 
 	char *Gets(char *strbuf, Size len)
 	{

source/common/filesystem/include/fs_filesystem.h

@@ -22,42 +22,6 @@ union LumpShortName
 };
 
 
-// A lump in memory.
-class FileData
-{
-public:
-	FileData() { lump = nullptr; }
-	const void *GetMem () { return lump->Cache; }
-	size_t GetSize () { return lump->LumpSize; }
-	const char* GetString () const { return (const char*)lump->Cache; }
-	const uint8_t* GetBytes() const { return (const uint8_t*)lump->Cache; }
-
-	FileData& operator = (const FileData& copy) = delete;
-
-	FileData(const FileData& copy)
-	{
-		lump = copy.lump;
-		lump->Lock();
-	}
-
-	~FileData()
-	{
-		if (lump) lump->Unlock();
-	}
-
-
-private:
-	FileData(FResourceLump* nlump)
-	{
-		lump = nlump;
-		if (lump) lump->Lock();
-	}
-
-	FResourceLump* lump;
-
-	friend class FileSystem;
-};
-
 struct FolderEntry
 {
 	const char *name;
@@ -129,9 +93,19 @@ public:
 	FileData ReadFile (const char *name) { return ReadFile (GetNumForName (name)); }
 	FileData ReadFileFullName(const char* name) { return ReadFile(GetNumForFullName(name)); }
 
-	FileReader OpenFileReader(int lump);		// opens a reader that redirects to the containing file's one.
+	FileReader OpenFileReader(int lump, int readertype, int readerflags);		// opens a reader that redirects to the containing file's one.
-	FileReader ReopenFileReader(int lump, bool alwayscache = false);		// opens an independent reader.
 	FileReader OpenFileReader(const char* name);
+	FileReader ReopenFileReader(const char* name, bool alwayscache = false);
+	FileReader OpenFileReader(int lump)
+	{
+		return OpenFileReader(lump, READER_SHARED, READERFLAG_SEEKABLE);
+	}
+
+	FileReader ReopenFileReader(int lump, bool alwayscache = false)
+	{
+		return OpenFileReader(lump, alwayscache ? READER_CACHED : READER_NEW, READERFLAG_SEEKABLE);
+	}
+
 
 	int FindLump (const char *name, int *lastlump, bool anyns=false);		// [RH] Find lumps with duplication
 	int FindLumpMulti (const char **names, int *lastlump, bool anyns = false, int *nameindex = NULL); // same with multiple possible names
@@ -145,8 +119,7 @@ public:
 
 	static uint32_t LumpNameHash (const char *name);		// [RH] Create hash key from an 8-char name
 
-	int FileLength (int lump) const;
+	ptrdiff_t FileLength (int lump) const;
-	int GetFileOffset (int lump);					// [RH] Returns offset of lump in the wadfile
 	int GetFileFlags (int lump);					// Return the flags for this lump
 	const char* GetFileShortName(int lump) const;
 	const char *GetFileFullName (int lump, bool returnshort = true) const;	// [RH] Returns the lump's full name

source/common/filesystem/include/fs_findfile.h

@@ -23,7 +23,6 @@ using FileList = std::vector<FileListEntry>;
 
 struct FCompressedBuffer;
 bool ScanDirectory(std::vector<FileListEntry>& list, const char* dirpath, const char* match, bool nosubdir = false, bool readhidden = false);
-bool WriteZip(const char* filename, const FCompressedBuffer* content, size_t contentcount);
 bool FS_DirEntryExists(const char* pathname, bool* isdir);
 
 inline void FixPathSeparator(char* path)

source/common/filesystem/include/resourcefile.h

@@ -7,6 +7,7 @@
 #include <vector>
 #include <string>
 #include "fs_files.h"
+#include "fs_decompress.h"
 
 namespace FileSys {
 	
@@ -18,7 +19,6 @@ void strReplace(std::string& str, const char* from, const char* to);
 struct LumpFilterInfo
 {
 	std::vector<std::string> gameTypeFilter;	// this can contain multiple entries
-	std::string dotFilter;
 
 	// The following are for checking if the root directory of a zip can be removed.
 	std::vector<std::string> reservedFolders;
@@ -78,125 +78,69 @@ typedef enum {
 
 enum ELumpFlags
 {
-	LUMPF_MAYBEFLAT = 1,	// might be a flat outside F_START/END
+	RESFF_MAYBEFLAT = 1,	// might be a flat inside a WAD outside F_START/END
-	LUMPF_FULLPATH = 2,		// contains a full path. This will trigger extended namespace checks when looking up short names.
+	RESFF_FULLPATH = 2,		// contains a full path. This will trigger extended namespace checks when looking up short names.
-	LUMPF_EMBEDDED = 4,		// marks an embedded resource file for later processing.
+	RESFF_EMBEDDED = 4,		// marks an embedded resource file for later processing.
-	LUMPF_SHORTNAME = 8,	// the stored name is a short extension-less name
+	RESFF_SHORTNAME = 8,	// the stored name is a short extension-less name
-	LUMPF_COMPRESSED = 16,	// compressed or encrypted, i.e. cannot be read with the container file's reader.
+	RESFF_COMPRESSED = 16,	// compressed or encrypted, i.e. cannot be read with the container file's reader.
+	RESFF_NEEDFILESTART = 32,	// The real position is not known yet and needs to be calculated on access
 };
 
-// This holds a compresed Zip entry with all needed info to decompress it.
+enum EReaderType
-struct FCompressedBuffer
 {
-	size_t mSize;
+	READER_SHARED = 0,	// returns a view into the parent's reader.
-	size_t mCompressedSize;
+	READER_NEW = 1,		// opens a new file handle
-	int mMethod;
+	READER_CACHED = 2,	// returns a MemoryArrayReader
-	int mZipFlags;
+	READERFLAG_SEEKABLE = 1	// ensure the reader is seekable.
-	unsigned mCRC32;
-	char *mBuffer;
-	const char* filename;
-
-	bool Decompress(char *destbuffer);
-	void Clean()
-	{
-		mSize = mCompressedSize = 0;
-		if (mBuffer != nullptr)
-		{
-			delete[] mBuffer;
-			mBuffer = nullptr;
-		}
-	}
 };
 
-struct FResourceLump
+struct FResourceEntry
 {
-protected:
+	size_t Length;
-	friend class FResourceFile;
+	size_t CompressedSize;
-	friend class FWadFile;	// this still needs direct access.
+	const char* FileName;
-	friend class FileData;
+	size_t Position;
-	friend class FileSystem;
+	int ResourceID;
-	friend class FLumpFile;
+	uint32_t CRC32;
-	friend class FLumpReader;
+	uint16_t Flags;
-	friend class FGrpFile;
+	uint16_t Method;
-	friend class F7ZFile;
+	int16_t Namespace;
-	friend class FSSIFile;
-	friend class FWHResFile;
-	friend class FZipFile;
-	friend class FPakFile;
-	friend class FRFFFile;
-	friend class FDirectory;
-	friend int lumpcmp(const void* a, const void* b);
-
-
-	int				LumpSize;
-	int				RefCount;
-//protected:
-	const char*		FullName;
-//public:
-	uint8_t			Flags;
-	char *			Cache;
-	FResourceFile *	Owner;
-
-public:
-	FResourceLump()
-	{
-		Cache = NULL;
-		Owner = NULL;
-		Flags = 0;
-		RefCount = 0;
-		FullName = "";
-		LumpSize = 0;
-	}
-	virtual ~FResourceLump();
-
-protected:
-
-	virtual FileReader *GetReader();
-	virtual FileReader NewReader();
-	virtual int GetFileOffset() { return -1; }
-	virtual int GetIndexNum() const { return -1; }
-	virtual int GetNamespace() const { return 0; }
-	void LumpNameSetup(const char* iname, StringPool* allocator);
-	void CheckEmbedded(LumpFilterInfo* lfi);
-	virtual FCompressedBuffer GetRawData();
-
-	void *Lock(); // validates the cache and increases the refcount.
-	int Unlock(); // decreases the refcount and frees the buffer
-
-	unsigned Size() const{ return LumpSize; }
-	int LockCount() const { return RefCount; }
-	const char* getName() { return FullName; }
-	void clearName() { FullName = ""; }
-
-protected:
-	virtual int FillCache() { return -1; }
-
 };
 
+void SetMainThread();
+
 class FResourceFile
 {
 public:
+	FResourceFile(const char* filename, StringPool* sp);
+	FResourceFile(const char* filename, FileReader& r, StringPool* sp);
+	const char* NormalizeFileName(const char* fn, int fallbackcp = 0);
+	FResourceEntry* AllocateEntries(int count);
+	void GenerateHash();
+	void PostProcessArchive(LumpFilterInfo* filter);
 protected:
 	FileReader Reader;
 	const char* FileName;
+	FResourceEntry* Entries = nullptr;
 	uint32_t NumLumps;
 	char Hash[48];
 	StringPool* stringpool;
 
-	FResourceFile(const char *filename, StringPool* sp);
-	FResourceFile(const char *filename, FileReader &r, StringPool* sp);
-
 	// for archives that can contain directories
-	void GenerateHash();
+	virtual void SetEntryAddress(uint32_t entry)
-	void PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterInfo *filter);
+	{
+		Entries[entry].Flags &= ~RESFF_NEEDFILESTART;
+	}
+	bool IsFileInFolder(const char* const resPath);
+	void CheckEmbedded(uint32_t entry, LumpFilterInfo* lfi);
 
 private:
 	uint32_t FirstLump;
 
-	int FilterLumps(const std::string& filtername, void *lumps, size_t lumpsize, uint32_t max);
+	int FilterLumps(const std::string& filtername, uint32_t max);
-	int FilterLumpsByGameType(LumpFilterInfo *filter, void *lumps, size_t lumpsize, uint32_t max);
+	bool FindPrefixRange(const char* filter, uint32_t max, uint32_t &start, uint32_t &end);
-	bool FindPrefixRange(const char* filter, void *lumps, size_t lumpsize, uint32_t max, uint32_t &start, uint32_t &end);
+	void JunkLeftoverFilters(uint32_t max);
-	void JunkLeftoverFilters(void *lumps, size_t lumpsize, uint32_t max);
+	void FindCommonFolder(LumpFilterInfo* filter);
 	static FResourceFile *DoOpenResourceFile(const char *filename, FileReader &file, bool containeronly, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 
 public:
@@ -211,47 +155,45 @@ public:
 	void SetFirstLump(uint32_t f) { FirstLump = f; }
 	const char* GetHash() const { return Hash; }
 
-	virtual FResourceLump *GetLump(int no) = 0;
-
 	int EntryCount() const { return NumLumps; }
 	int FindEntry(const char* name);
 
-	size_t Length(int entry)
+	size_t Length(uint32_t entry)
 	{
-		auto l = GetLump(entry);
+		return (entry < NumLumps) ? Entries[entry].Length : 0;
-		return l ? l->LumpSize : -1;
+	}
+	size_t Offset(uint32_t entry)
+	{
+		return (entry < NumLumps) ? Entries[entry].Position : 0;
 	}
 
-	FileReader GetEntryReader(int entry, bool newreader = true)
+	// default is the safest reader type.
+	virtual FileReader GetEntryReader(uint32_t entry, int readertype = READER_NEW, int flags = READERFLAG_SEEKABLE);
+
+	int GetEntryFlags(uint32_t entry)
 	{
-		auto l = GetLump(entry);
+		return (entry < NumLumps) ? Entries[entry].Flags : 0;
-		return l ? l->NewReader() : FileReader();
 	}
 
-	int GetEntryFlags(int entry)
+	int GetEntryNamespace(uint32_t entry)
 	{
-		auto l = GetLump(entry);
+		return (entry < NumLumps) ? Entries[entry].Namespace : ns_hidden;
-		return l ? l->Flags : 0;
 	}
 
-	ResourceData Read(int entry)
+	int GetEntryResourceID(uint32_t entry)
 	{
-		auto fr = GetEntryReader(entry, false);
+		return (entry < NumLumps) ? Entries[entry].ResourceID : -1;
-		return fr.Read();
 	}
 
-	const char* getName(int entry)
+	const char* getName(uint32_t entry)
 	{
-		auto l = GetLump(entry);
+		return (entry < NumLumps) ? Entries[entry].FileName : nullptr;
-		return l ? l->FullName : nullptr;
-	}
-	FCompressedBuffer GetRawData(int entry)
-	{
-		auto l = GetLump(entry);
-		if (!l) return {};
-		return l->GetRawData();
 	}
 
+	virtual FileData Read(int entry);
+
+	virtual FCompressedBuffer GetRawData(uint32_t entry);
+
 	FileReader Destroy()
 	{
 		auto fr = std::move(Reader);

source/common/filesystem/include/w_zip.h

@@ -7,7 +7,8 @@
 #define FORCE_PACKED
 #endif
 
-namespace FileSys {
+#include <stdint.h>
+
 	
 #pragma pack(1)
 // FZipCentralInfo
@@ -107,5 +108,4 @@ struct FZipLocalFileHeader
 // File header flags.
 #define ZF_ENCRYPTED			0x1
 
-}
 #endif

source/common/filesystem/source/ancientzip.cpp

@@ -44,6 +44,7 @@
 
 
 #include <stdlib.h>
+#include <assert.h>
 #include "ancientzip.h"
 
 namespace FileSys {
@@ -125,7 +126,7 @@ static const unsigned char BitReverse4[] = {
 #define FIRST_BIT_LEN	8
 #define REST_BIT_LEN	4
 
-void FZipExploder::InsertCode(TArray<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value)
+void FZipExploder::InsertCode(std::vector<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value)
 {
 	assert(len > 0);
 	unsigned int node = pos + (code & ((1 << bits) - 1));
@@ -161,12 +162,12 @@ void FZipExploder::InsertCode(TArray<HuffNode> &decoder, unsigned int pos, int b
 	}
 }
 
-unsigned int FZipExploder::InitTable(TArray<HuffNode> &decoder, int numspots)
+unsigned int FZipExploder::InitTable(std::vector<HuffNode> &decoder, int numspots)
 {
-	unsigned int start = decoder.Size();
+	size_t start = decoder.size();
-	decoder.Reserve(numspots);
+	decoder.resize(decoder.size() + numspots);
 	memset(&decoder[start], 0, sizeof(HuffNode)*numspots);
-	return start;
+	return (unsigned)start;
 }
 
 int FZipExploder::buildercmp(const void *a, const void *b)
@@ -180,7 +181,7 @@ int FZipExploder::buildercmp(const void *a, const void *b)
 	return d;
 }
 
-int FZipExploder::BuildDecoder(TArray<HuffNode> &decoder, TableBuilder *values, int numvals)
+int FZipExploder::BuildDecoder(std::vector<HuffNode> &decoder, TableBuilder *values, int numvals)
 {
 	int i;
 
@@ -218,7 +219,7 @@ int FZipExploder::BuildDecoder(TArray<HuffNode> &decoder, TableBuilder *values,
 }
 
 
-int FZipExploder::DecodeSFValue(const TArray<HuffNode> &decoder)
+int FZipExploder::DecodeSFValue(const std::vector<HuffNode> &decoder)
 {
 	unsigned int bits = FIRST_BIT_LEN, table = 0, code;
 	const HuffNode *pos;
@@ -236,7 +237,7 @@ int FZipExploder::DecodeSFValue(const TArray<HuffNode> &decoder)
 }
 
 
-int FZipExploder::DecodeSF(TArray<HuffNode> &decoder, int numvals)
+int FZipExploder::DecodeSF(std::vector<HuffNode> &decoder, int numvals)
 {
 	TableBuilder builder[256];
 	unsigned char a, c;

source/common/filesystem/source/ancientzip.h

@@ -27,18 +27,18 @@ class FZipExploder
 		unsigned short Code;
 	};
 
-	TArray<HuffNode> LiteralDecoder;
+	std::vector<HuffNode> LiteralDecoder;
-	TArray<HuffNode> DistanceDecoder;
+	std::vector<HuffNode> DistanceDecoder;
-	TArray<HuffNode> LengthDecoder;
+	std::vector<HuffNode> LengthDecoder;
 	unsigned char ReadBuf[256];
 	unsigned int bs, be;
 
 	static int buildercmp(const void *a, const void *b);
-	void InsertCode(TArray<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value);
+	void InsertCode(std::vector<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value);
-	unsigned int InitTable(TArray<HuffNode> &decoder, int numspots);
+	unsigned int InitTable(std::vector<HuffNode> &decoder, int numspots);
-	int BuildDecoder(TArray<HuffNode> &decoder, TableBuilder *values, int numvals);
+	int BuildDecoder(std::vector<HuffNode> &decoder, TableBuilder *values, int numvals);
-	int DecodeSFValue(const TArray<HuffNode> &currentTree);
+	int DecodeSFValue(const std::vector<HuffNode> &currentTree);
-	int DecodeSF(TArray<HuffNode> &decoder, int numvals);
+	int DecodeSF(std::vector<HuffNode> &decoder, int numvals);
 public:
 	int Explode(unsigned char *out, unsigned int outsize, FileReader &in, unsigned int insize, int flags);
 };

source/common/filesystem/source/critsec.cpp

@@ -0,0 +1,150 @@
+/*
+**
+**
+**---------------------------------------------------------------------------
+** Copyright 2005-2016 Randy Heit
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+**
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**---------------------------------------------------------------------------
+**
+*/
+
+namespace FileSys {
+
+#ifdef _WIN32
+
+#ifndef _WINNT_
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#endif
+
+class FInternalCriticalSection
+{
+public:
+	void Enter()
+	{
+		AcquireSRWLockExclusive(&CritSec);
+	}
+	void Leave()
+	{
+		ReleaseSRWLockExclusive(&CritSec);
+	}
+private:
+	SRWLOCK CritSec = SRWLOCK_INIT;
+};
+
+
+FInternalCriticalSection *CreateCriticalSection()
+{
+	return new FInternalCriticalSection();
+}
+
+void DeleteCriticalSection(FInternalCriticalSection *c)
+{
+	delete c;
+}
+
+void EnterCriticalSection(FInternalCriticalSection *c)
+{
+	c->Enter();
+}
+
+void LeaveCriticalSection(FInternalCriticalSection *c)
+{
+	c->Leave();
+}
+
+#else
+
+#include "critsec.h"
+
+#include <pthread.h>
+
+class FInternalCriticalSection
+{
+public:
+	FInternalCriticalSection();
+	~FInternalCriticalSection();
+
+	void Enter();
+	void Leave();
+
+private:
+	pthread_mutex_t m_mutex;
+
+};
+
+// TODO: add error handling
+
+FInternalCriticalSection::FInternalCriticalSection()
+{
+	pthread_mutexattr_t attributes;
+	pthread_mutexattr_init(&attributes);
+	pthread_mutexattr_settype(&attributes, PTHREAD_MUTEX_RECURSIVE);
+
+	pthread_mutex_init(&m_mutex, &attributes);
+
+	pthread_mutexattr_destroy(&attributes);
+}
+
+FInternalCriticalSection::~FInternalCriticalSection()
+{
+	pthread_mutex_destroy(&m_mutex);
+}
+
+void FInternalCriticalSection::Enter()
+{
+	pthread_mutex_lock(&m_mutex);
+}
+
+void FInternalCriticalSection::Leave()
+{
+	pthread_mutex_unlock(&m_mutex);
+}
+
+
+FInternalCriticalSection *CreateCriticalSection()
+{
+	return new FInternalCriticalSection();
+}
+
+void DeleteCriticalSection(FInternalCriticalSection *c)
+{
+	delete c;
+}
+
+void EnterCriticalSection(FInternalCriticalSection *c)
+{
+	c->Enter();
+}
+
+void LeaveCriticalSection(FInternalCriticalSection *c)
+{
+	c->Leave();
+} 
+
+#endif
+
+}

source/common/filesystem/source/critsec.h

@@ -0,0 +1,39 @@
+#pragma once
+
+namespace FileSys {
+// System independent critical sections without polluting the namespace with the operating system headers.
+class FInternalCriticalSection;
+FInternalCriticalSection *CreateCriticalSection();
+void DeleteCriticalSection(FInternalCriticalSection *c);
+void EnterCriticalSection(FInternalCriticalSection *c);
+void LeaveCriticalSection(FInternalCriticalSection *c);
+
+// This is just a convenience wrapper around the function interface adjusted to use std::lock_guard
+class FCriticalSection
+{
+public:
+	FCriticalSection()
+	{
+		c = CreateCriticalSection();
+	}
+	
+	~FCriticalSection()
+	{
+		DeleteCriticalSection(c);
+	}
+
+	void lock()
+	{
+		EnterCriticalSection(c);
+	}
+	
+	void unlock()
+	{
+		LeaveCriticalSection(c);
+	}
+
+private:
+	FInternalCriticalSection *c;
+
+};
+}

source/common/filesystem/source/file_7z.cpp

@@ -38,6 +38,9 @@
 #include "7zCrc.h"
 #include "resourcefile.h"
 #include "fs_findfile.h"
+#include "unicode.h"
+#include "critsec.h"
+#include <mutex>
 
 
 namespace FileSys {
@@ -158,20 +161,6 @@ struct C7zArchive
 		return res;
 	}
 };
-//==========================================================================
-//
-// Zip Lump
-//
-//==========================================================================
-
-struct F7ZLump : public FResourceLump
-{
-	int		Position;
-
-	virtual int FillCache() override;
-
-};
-
 
 //==========================================================================
 //
@@ -183,14 +172,15 @@ class F7ZFile : public FResourceFile
 {
 	friend struct F7ZLump;
 
-	F7ZLump *Lumps;
 	C7zArchive *Archive;
+	FCriticalSection critsec;
 
 public:
 	F7ZFile(const char * filename, FileReader &filer, StringPool* sp);
 	bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf);
 	virtual ~F7ZFile();
-	virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; }
+	FileData Read(int entry) override;
+	FileReader GetEntryReader(uint32_t entry, int, int) override;
 };
 
 
@@ -204,8 +194,7 @@ public:
 F7ZFile::F7ZFile(const char * filename, FileReader &filer, StringPool* sp)
 	: FResourceFile(filename, filer, sp) 
 {
-	Lumps = NULL;
+	Archive = nullptr;
-	Archive = NULL;
 }
 
 
@@ -247,12 +236,11 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
 
 	CSzArEx* const archPtr = &Archive->DB;
 
+	AllocateEntries(archPtr->NumFiles);
 	NumLumps = archPtr->NumFiles;
-	Lumps = new F7ZLump[NumLumps];
 
-	F7ZLump *lump_p = Lumps;
 	std::u16string nameUTF16;
-	std::string nameASCII;
+	std::vector<char> nameASCII;
 
 	for (uint32_t i = 0; i < NumLumps; ++i)
 	{
@@ -273,21 +261,17 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
 
 		nameUTF16.resize((unsigned)nameLength);
 		nameASCII.resize((unsigned)nameLength);
-		// note that the file system is not equipped to handle non-ASCII, so don't bother with proper Unicode conversion here.
-		SzArEx_GetFileNameUtf16(archPtr, i, (UInt16*)nameUTF16.data());
-		for (size_t c = 0; c < nameLength; ++c)
-		{
-			nameASCII[c] = tolower(static_cast<char>(nameUTF16[c]));
-		}
-		FixPathSeparator(&nameASCII.front());
 
-		lump_p->LumpNameSetup(nameASCII.c_str(), stringpool);
+		SzArEx_GetFileNameUtf16(archPtr, i, (UInt16*)nameUTF16.data());
-		lump_p->LumpSize = static_cast<int>(SzArEx_GetFileSize(archPtr, i));
+		utf16_to_utf8((uint16_t*)nameUTF16.data(), nameASCII);
-		lump_p->Owner = this;
+
-		lump_p->Flags = LUMPF_FULLPATH|LUMPF_COMPRESSED;
+		Entries[i].FileName = NormalizeFileName(nameASCII.data());
-		lump_p->Position = i;
+		Entries[i].Length = SzArEx_GetFileSize(archPtr, i);
-		lump_p->CheckEmbedded(filter);
+		Entries[i].Flags = RESFF_FULLPATH|RESFF_COMPRESSED;
-		lump_p++;
+		Entries[i].ResourceID = -1;
+		Entries[i].Namespace = ns_global;
+		Entries[i].Method = METHOD_INVALID;
+		Entries[i].Position = i;
 	}
 	// Resize the lump record array to its actual size
 	NumLumps -= skipped;
@@ -296,10 +280,9 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
 	{
 		// Quick check for unsupported compression method
 
-		TArray<char> temp;
+		FileData temp(nullptr, Entries[0].Length);
-		temp.Resize(Lumps[0].LumpSize);
 
-		if (SZ_OK != Archive->Extract(Lumps[0].Position, &temp[0]))
+		if (SZ_OK != Archive->Extract(Entries[0].Position, (char*)temp.writable()))
 		{
 			Printf(FSMessageLevel::Error, "%s: unsupported 7z/LZMA file!\n", FileName);
 			return false;
@@ -307,7 +290,7 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
 	}
 
 	GenerateHash();
-	PostProcessArchive(&Lumps[0], sizeof(F7ZLump), filter);
+	PostProcessArchive(filter);
 	return true;
 }
 
@@ -319,11 +302,7 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
 
 F7ZFile::~F7ZFile()
 {
-	if (Lumps != NULL)
+	if (Archive != nullptr)
-	{
-		delete[] Lumps;
-	}
-	if (Archive != NULL)
 	{
 		delete Archive;
 	}
@@ -331,22 +310,41 @@ F7ZFile::~F7ZFile()
 
 //==========================================================================
 //
-// Fills the lump cache and performs decompression
+// Reads data for one entry into a buffer
 //
 //==========================================================================
 
-int F7ZLump::FillCache()
+FileData F7ZFile::Read(int entry)
 {
-	Cache = new char[LumpSize];
+	FileData buffer;
-	SRes code = static_cast<F7ZFile*>(Owner)->Archive->Extract(Position, Cache);
+	if ((entry >= 0 || entry < NumLumps) && Entries[entry].Length > 0)
-	if (code != SZ_OK)
 	{
-		throw FileSystemException("Error %d reading from 7z archive", code);
+		auto p = buffer.allocate(Entries[entry].Length);
+		// There is no realistic way to keep multiple references to a 7z file open without massive overhead so to make this thread-safe a mutex is the only option.
+		std::lock_guard<FCriticalSection> lock(critsec);
+		SRes code = Archive->Extract(Entries[entry].Position, (char*)p);
+		if (code != SZ_OK) buffer.clear();
 	}
-	RefCount = 1;
+	return buffer;
-	return 1;
 }
 
+//==========================================================================
+//
+// This can only return a FileReader to a memory buffer.
+//
+//==========================================================================
+
+FileReader F7ZFile::GetEntryReader(uint32_t entry, int, int)
+{
+	FileReader fr;
+	if (entry < 0 || entry >= NumLumps) return fr;
+	auto buffer = Read(entry);
+	if (buffer.size() > 0)
+		fr.OpenMemoryArray(buffer);
+	return fr;
+}
+
+
 //==========================================================================
 //
 // File open

source/common/filesystem/source/file_directory.cpp

@@ -48,21 +48,6 @@ std::string FS_FullPath(const char* directory);
 std::wstring toWide(const char* str);
 #endif
 
-//==========================================================================
-//
-// Zip Lump
-//
-//==========================================================================
-
-struct FDirectoryLump : public FResourceLump
-{
-	FileReader NewReader() override;
-	int FillCache() override;
-
-	const char* mFullPath;
-};
-
-
 //==========================================================================
 //
 // Zip file
@@ -71,16 +56,15 @@ struct FDirectoryLump : public FResourceLump
 
 class FDirectory : public FResourceFile
 {
-	TArray<FDirectoryLump> Lumps;
 	const bool nosubdir;
+	const char* mBasePath;
 
 	int AddDirectory(const char* dirpath, LumpFilterInfo* filter, FileSystemMessageFunc Printf);
-	void AddEntry(const char *fullpath, const char* relpath, int size);
 
 public:
 	FDirectory(const char * dirname, StringPool* sp, bool nosubdirflag = false);
 	bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf);
-	virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; }
+	FileReader GetEntryReader(uint32_t entry, int, int) override;
 };
 
 
@@ -116,8 +100,17 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy
 	}
 	else
 	{
+		mBasePath = nullptr;
+		AllocateEntries(list.size());
 		for(auto& entry : list)
 		{
+			if (mBasePath == nullptr)
+			{
+				// extract the base path from the first entry to cover changes made in ScanDirectory.
+				auto full = entry.FilePath.find(entry.FilePathRel);
+				std::string path(entry.FilePath, 0, full);
+				mBasePath = stringpool->Strdup(path.c_str());
+			}
 			if (!entry.isDirectory)
 			{
 				auto fi = entry.FileName;
@@ -128,6 +121,7 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy
 					continue;
 				}
 
+
 				if (filter->filenamecheck == nullptr || filter->filenamecheck(fi.c_str(), entry.FilePath.c_str()))
 				{
 					if (entry.Length > 0x7fffffff)
@@ -135,7 +129,13 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy
 						Printf(FSMessageLevel::Warning, "%s is larger than 2GB and will be ignored\n", entry.FilePath.c_str());
 						continue;
 					}
-					AddEntry(entry.FilePath.c_str(), entry.FilePathRel.c_str(), (int)entry.Length);
+					// for internal access we use the normalized form of the relative path.
+					Entries[count].FileName = NormalizeFileName(entry.FilePathRel.c_str());
+					Entries[count].Length = entry.Length;
+					Entries[count].Flags = RESFF_FULLPATH;
+					Entries[count].ResourceID = -1;
+					Entries[count].Method = METHOD_STORED;
+					Entries[count].Namespace = ns_global;
 					count++;
 				}
 			}
@@ -153,7 +153,6 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy
 bool FDirectory::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 {
 	NumLumps = AddDirectory(FileName, filter, Printf);
-	PostProcessArchive(&Lumps[0], sizeof(FDirectoryLump), filter);
 	return true;
 }
 
@@ -163,62 +162,22 @@ bool FDirectory::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 //
 //==========================================================================
 
-void FDirectory::AddEntry(const char *fullpath, const char* relpath, int size)
+FileReader FDirectory::GetEntryReader(uint32_t entry, int readertype, int)
-{
-	FDirectoryLump *lump_p = &Lumps[Lumps.Reserve(1)];
-
-	// Store the full path here so that we can access the file later, even if it is from a filter directory.
-	lump_p->mFullPath = stringpool->Strdup(fullpath);
-
-	// [mxd] Convert name to lowercase
-	std::string name = relpath;
-	for (auto& c : name) c = tolower(c);
-
-	// The lump's name is only the part relative to the main directory
-	lump_p->LumpNameSetup(name.c_str(), stringpool);
-	lump_p->LumpSize = size;
-	lump_p->Owner = this;
-	lump_p->Flags = 0;
-	lump_p->CheckEmbedded(nullptr);
-}
-
-
-//==========================================================================
-//
-//
-//
-//==========================================================================
-
-FileReader FDirectoryLump::NewReader()
 {
 	FileReader fr;
-	fr.OpenFile(mFullPath);
+	if (entry < NumLumps)
+	{
+		std::string fn = mBasePath; fn += Entries[entry].FileName;
+		fr.OpenFile(fn.c_str());
+		if (readertype == READER_CACHED)
+		{
+			auto data = fr.Read();
+			fr.OpenMemoryArray(data);
+		}
+	}
 	return fr;
 }
 
-//==========================================================================
-//
-//
-//
-//==========================================================================
-
-int FDirectoryLump::FillCache()
-{
-	FileReader fr;
-	Cache = new char[LumpSize];
-	if (!fr.OpenFile(mFullPath))
-	{
-		throw FileSystemException("unable to open file");
-	}
-	auto read = fr.Read(Cache, LumpSize);
-	if (read != LumpSize)
-	{
-		throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize);
-	}
-	RefCount = 1;
-	return 1;
-}
-
 //==========================================================================
 //
 // File open

source/common/filesystem/source/file_grp.cpp

@@ -33,7 +33,7 @@
 **
 */
 
-#include "resourcefile_internal.h"
+#include "resourcefile.h"
 #include "fs_swap.h"
 
 namespace FileSys {
@@ -65,62 +65,39 @@ struct GrpLump
 };
 
 
-//==========================================================================
-//
-// Build GRP file
-//
-//==========================================================================
-
-class FGrpFile : public FUncompressedFile
-{
-public:
-	FGrpFile(const char * filename, FileReader &file, StringPool* sp);
-	bool Open(LumpFilterInfo* filter);
-};
-
-
-//==========================================================================
-//
-// Initializes a Build GRP file
-//
-//==========================================================================
-
-FGrpFile::FGrpFile(const char *filename, FileReader &file, StringPool* sp)
-: FUncompressedFile(filename, file, sp)
-{
-}
-
 //==========================================================================
 //
 // Open it
 //
 //==========================================================================
 
-bool FGrpFile::Open(LumpFilterInfo* filter)
+static bool OpenGrp(FResourceFile* file, LumpFilterInfo* filter)
 {
 	GrpHeader header;
 
-	Reader.Read(&header, sizeof(header));
+	auto Reader = file->GetContainerReader();
-	NumLumps = LittleLong(header.NumLumps);
+	Reader->Read(&header, sizeof(header));
+	uint32_t NumLumps = LittleLong(header.NumLumps);
+	auto Entries = file->AllocateEntries(NumLumps);
 
 	GrpLump *fileinfo = new GrpLump[NumLumps];
-	Reader.Read (fileinfo, NumLumps * sizeof(GrpLump));
+	Reader->Read (fileinfo, NumLumps * sizeof(GrpLump));
-
-	Lumps.Resize(NumLumps);
 
 	int Position = sizeof(GrpHeader) + NumLumps * sizeof(GrpLump);
 
 	for(uint32_t i = 0; i < NumLumps; i++)
 	{
-		Lumps[i].Owner = this;
+		Entries[i].Position = Position;
-		Lumps[i].Position = Position;
+		Entries[i].Length = LittleLong(fileinfo[i].Size);
-		Lumps[i].LumpSize = LittleLong(fileinfo[i].Size);
 		Position += fileinfo[i].Size;
-		Lumps[i].Flags = 0;
+		Entries[i].Flags = 0;
+		Entries[i].Namespace = ns_global;
 		fileinfo[i].NameWithZero[12] = '\0';	// Be sure filename is null-terminated
-		Lumps[i].LumpNameSetup(fileinfo[i].NameWithZero, stringpool);
+		Entries[i].ResourceID = -1;
+		Entries[i].Method = METHOD_STORED;
+		Entries[i].FileName = file->NormalizeFileName(fileinfo[i].Name);
 	}
-	GenerateHash();
+	file->GenerateHash();
 	delete[] fileinfo;
 	return true;
 }
@@ -143,12 +120,12 @@ FResourceFile *CheckGRP(const char *filename, FileReader &file, LumpFilterInfo*
 		file.Seek(0, FileReader::SeekSet);
 		if (!memcmp(head, "KenSilverman", 12))
 		{
-			auto rf = new FGrpFile(filename, file, sp);
+			auto rf = new FResourceFile(filename, file, sp);
-			if (rf->Open(filter)) return rf;
+			if (OpenGrp(rf, filter)) return rf;
 			file = rf->Destroy();
 		}
 	}
-	return NULL;
+	return nullptr;
 }
 
 }

source/common/filesystem/source/file_hog.cpp

@@ -0,0 +1,109 @@
+/*
+** file_hog.cpp
+**
+** reads Descent .hog files
+**
+**---------------------------------------------------------------------------
+** Copyright 2023 Christoph Oelckers
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+**
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**---------------------------------------------------------------------------
+**
+**
+*/
+
+
+#include "resourcefile.h"
+#include "fs_swap.h"
+
+namespace FileSys {
+    using namespace byteswap;
+
+
+
+
+static bool OpenHog(FResourceFile* rf, LumpFilterInfo* filter)
+{
+    auto Reader = rf->GetContainerReader();
+    FileReader::Size length = Reader->GetLength();
+
+    std::vector<FResourceEntry> entries;
+    // Hogs store their data as a list of file records, each containing a name, length and the actual data.
+    // To read the directory the entire file must be scanned.
+    while (Reader->Tell() <= length)
+    {
+        char name[13];
+
+        auto r = Reader->Read(&name, 13);
+        if (r < 13) break;
+        name[12] = 0;
+        uint32_t elength = Reader->ReadUInt32();
+
+        FResourceEntry Entry;
+        Entry.Position = Reader->Tell();
+        Entry.Length = elength;
+        Entry.Flags = 0;
+        Entry.CRC32 = 0;
+        Entry.Namespace = ns_global;
+        Entry.ResourceID = -1;
+        Entry.Method = METHOD_STORED;
+        Entry.FileName = rf->NormalizeFileName(name);
+        entries.push_back(Entry);
+        Reader->Seek(elength, FileReader::SeekCur);
+    }
+    auto Entries = rf->AllocateEntries((int)entries.size());
+    memcpy(Entries, entries.data(), entries.size() * sizeof(Entries[0]));
+    rf->GenerateHash();
+    return true;
+}
+
+
+//==========================================================================
+//
+// File open
+//
+//==========================================================================
+
+FResourceFile* CheckHog(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp)
+{
+    char head[3];
+
+    if (file.GetLength() >= 20)
+    {
+        file.Seek(0, FileReader::SeekSet);
+        file.Read(&head, 3);
+        if (!memcmp(head, "DHF", 3))
+        {
+            auto rf = new FResourceFile(filename, file, sp);
+            if (OpenHog(rf, filter)) return rf;
+            file = rf->Destroy();
+        }
+        file.Seek(0, FileReader::SeekSet);
+    }
+    return nullptr;
+}
+
+
+}

source/common/filesystem/source/file_lump.cpp

@@ -32,49 +32,25 @@
 **
 */
 
-#include "resourcefile_internal.h"
+#include "resourcefile.h"
 
 namespace FileSys {
-//==========================================================================
-//
-// Single lump
-//
-//==========================================================================
-
-class FLumpFile : public FUncompressedFile
-{
-public:
-	FLumpFile(const char * filename, FileReader &file, StringPool* sp);
-	bool Open(LumpFilterInfo* filter);
-};
-
-
-//==========================================================================
-//
-// FLumpFile::FLumpFile
-//
-//==========================================================================
-
-FLumpFile::FLumpFile(const char *filename, FileReader &file, StringPool* sp)
-	: FUncompressedFile(filename, file, sp)
-{
-}
-
 //==========================================================================
 //
 // Open it
 //
 //==========================================================================
 
-bool FLumpFile::Open(LumpFilterInfo*)
+static bool OpenLump(FResourceFile* file, LumpFilterInfo*)
 {
-	Lumps.Resize(1);
+	auto Entries = file->AllocateEntries(1);
-	Lumps[0].LumpNameSetup(ExtractBaseName(FileName, true).c_str(), stringpool);
+	Entries[0].FileName = file->NormalizeFileName(ExtractBaseName(file->GetFileName(), true).c_str());
-	Lumps[0].Owner = this;
+	Entries[0].Namespace = ns_global;
-	Lumps[0].Position = 0;
+	Entries[0].ResourceID = -1;
-	Lumps[0].LumpSize = (int)Reader.GetLength();
+	Entries[0].Position = 0;
-	Lumps[0].Flags = 0;
+	Entries[0].Length = file->GetContainerReader()->GetLength();
-	NumLumps = 1;
+	Entries[0].Method = METHOD_STORED;
+	Entries[0].Flags = 0;
 	return true;
 }
 
@@ -87,8 +63,8 @@ bool FLumpFile::Open(LumpFilterInfo*)
 FResourceFile *CheckLump(const char *filename, FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp)
 {
 	// always succeeds
-	auto rf = new FLumpFile(filename, file, sp);
+	auto rf = new FResourceFile(filename, file, sp);
-	if (rf->Open(filter)) return rf;
+	if (OpenLump(rf, filter)) return rf;
 	file = rf->Destroy();
 	return NULL;
 }

source/common/filesystem/source/file_mvl.cpp

@@ -0,0 +1,98 @@
+/*
+** file_mvl.cpp
+**
+** reads Descent2 .mvl files
+**
+**---------------------------------------------------------------------------
+** Copyright 2023 Christoph Oelckers
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+**
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**---------------------------------------------------------------------------
+**
+**
+*/
+
+
+#include "resourcefile.h"
+#include "fs_swap.h"
+
+namespace FileSys {
+    using namespace byteswap;
+
+
+
+static bool OpenMvl(FResourceFile* rf, LumpFilterInfo* filter)
+{
+    auto Reader = rf->GetContainerReader();
+    auto count = Reader->ReadUInt32();
+    auto Entries = rf->AllocateEntries(count);
+    size_t pos = 8 + (17 * count);   // files start after the directory
+
+    for (uint32_t i = 0; i < count; i++)
+    {
+        char name[13];
+        Reader->Read(&name, 13);
+        name[12] = 0;
+        uint32_t elength = Reader->ReadUInt32();
+
+        Entries[i].Position = pos;
+        Entries[i].Length = elength;
+        Entries[i].ResourceID = -1;
+        Entries[i].FileName = rf->NormalizeFileName(name);
+
+        pos += elength;
+    }
+
+    return true;
+}
+
+
+//==========================================================================
+//
+// File open
+//
+//==========================================================================
+
+FResourceFile* CheckMvl(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp)
+{
+    char head[4];
+
+    if (file.GetLength() >= 20)
+    {
+        file.Seek(0, FileReader::SeekSet);
+        file.Read(&head, 4);
+        if (!memcmp(head, "DMVL", 4))
+        {
+            auto rf = new FResourceFile(filename, file, sp);
+            if (OpenMvl(rf, filter)) return rf;
+            file = rf->Destroy();
+        }
+        file.Seek(0, FileReader::SeekSet);
+    }
+    return nullptr;
+}
+
+
+}

source/common/filesystem/source/file_pak.cpp

@@ -32,7 +32,7 @@
 **
 */
 
-#include "resourcefile_internal.h"
+#include "resourcefile.h"
 
 namespace FileSys {
 
@@ -57,64 +57,38 @@ struct dpackheader_t
 } ;
 
 
-//==========================================================================
-//
-// Wad file
-//
-//==========================================================================
-
-class FPakFile : public FUncompressedFile
-{
-public:
-	FPakFile(const char * filename, FileReader &file, StringPool* sp);
-	bool Open(LumpFilterInfo* filter);
-};
-
-
-//==========================================================================
-//
-// FWadFile::FWadFile
-//
-// Initializes a WAD file
-//
-//==========================================================================
-
-FPakFile::FPakFile(const char *filename, FileReader &file, StringPool* sp)
-	: FUncompressedFile(filename, file, sp)
-{
-}
-
 //==========================================================================
 //
 // Open it
 //
 //==========================================================================
 
-bool FPakFile::Open(LumpFilterInfo* filter)
+static bool OpenPak(FResourceFile* file, LumpFilterInfo* filter)
 {
 	dpackheader_t header;
 
-	Reader.Read(&header, sizeof(header));
+	auto Reader = file->GetContainerReader();
-	NumLumps = LittleLong(header.dirlen) / sizeof(dpackfile_t);
+	Reader->Read(&header, sizeof(header));
+	uint32_t NumLumps = header.dirlen / sizeof(dpackfile_t);
+	auto Entries = file->AllocateEntries(NumLumps);
 	header.dirofs = LittleLong(header.dirofs);
 
-	TArray<dpackfile_t> fileinfo(NumLumps, true);
+	Reader->Seek (header.dirofs, FileReader::SeekSet);
-	Reader.Seek (header.dirofs, FileReader::SeekSet);
+	auto fd = Reader->Read (NumLumps * sizeof(dpackfile_t));
-	Reader.Read (fileinfo.Data(), NumLumps * sizeof(dpackfile_t));
+	auto fileinfo = (const dpackfile_t*)fd.data();
-
-	Lumps.Resize(NumLumps);
 
 	for(uint32_t i = 0; i < NumLumps; i++)
 	{
-		Lumps[i].LumpNameSetup(fileinfo[i].name, stringpool);
+		Entries[i].Position = LittleLong(fileinfo[i].filepos);
-		Lumps[i].Flags = LUMPF_FULLPATH;
+		Entries[i].Length = LittleLong(fileinfo[i].filelen);
-		Lumps[i].Owner = this;
+		Entries[i].Flags = RESFF_FULLPATH;
-		Lumps[i].Position = LittleLong(fileinfo[i].filepos);
+		Entries[i].Namespace = ns_global;
-		Lumps[i].LumpSize = LittleLong(fileinfo[i].filelen);
+		Entries[i].ResourceID = -1;
-		Lumps[i].CheckEmbedded(filter);
+		Entries[i].Method = METHOD_STORED;
+		Entries[i].FileName = file->NormalizeFileName(fileinfo[i].name);
 	}
-	GenerateHash();
+	file->GenerateHash();
-	PostProcessArchive(&Lumps[0], sizeof(Lumps[0]), filter);
+	file->PostProcessArchive(filter);
 	return true;
 }
 
@@ -136,8 +110,8 @@ FResourceFile *CheckPak(const char *filename, FileReader &file, LumpFilterInfo*
 		file.Seek(0, FileReader::SeekSet);
 		if (!memcmp(head, "PACK", 4))
 		{
-			auto rf = new FPakFile(filename, file, sp);
+			auto rf = new FResourceFile(filename, file, sp);
-			if (rf->Open(filter)) return rf;
+			if (OpenPak(rf, filter)) return rf;
 			file = rf->Destroy();
 		}
 	}

source/common/filesystem/source/file_rff.cpp

@@ -33,7 +33,8 @@
 **
 */
 
-#include "resourcefile_internal.h"
+#include <assert.h>
+#include "resourcefile.h"
 #include "fs_swap.h"
 
 namespace FileSys {
@@ -67,22 +68,6 @@ struct RFFLump
 	uint32_t		IndexNum;	// Used by .sfx, possibly others
 };
 
-//==========================================================================
-//
-// Blood RFF lump (uncompressed lump with encryption)
-//
-//==========================================================================
-
-struct FRFFLump : public FUncompressedLump
-{
-	virtual FileReader *GetReader();
-	virtual int FillCache() override;
-
-	uint32_t		IndexNum;
-
-	int GetIndexNum() const { return IndexNum; }
-};
-
 //==========================================================================
 //
 // BloodCrypt
@@ -100,68 +85,47 @@ void BloodCrypt (void *data, int key, int len)
 }
 
 
-//==========================================================================
-//
-// Blood RFF file
-//
-//==========================================================================
-
-class FRFFFile : public FResourceFile
-{
-	FRFFLump *Lumps;
-
-public:
-	FRFFFile(const char * filename, FileReader &file, StringPool* sp);
-	virtual ~FRFFFile();
-	virtual bool Open(LumpFilterInfo* filter);
-	virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; }
-};
-
-
 //==========================================================================
 //
 // Initializes a Blood RFF file
 //
 //==========================================================================
 
-FRFFFile::FRFFFile(const char *filename, FileReader &file, StringPool* sp)
+static bool OpenRFF(FResourceFile* file, LumpFilterInfo*)
-: FResourceFile(filename, file, sp)
-{
-	Lumps = NULL;
-}
-
-//==========================================================================
-//
-// Initializes a Blood RFF file
-//
-//==========================================================================
-
-bool FRFFFile::Open(LumpFilterInfo*)
 {
 	RFFLump *lumps;
 	RFFInfo header;
 
-	Reader.Read(&header, sizeof(header));
+	auto Reader = file->GetContainerReader();
+	Reader->Read(&header, sizeof(header));
 
-	NumLumps = LittleLong(header.NumLumps);
+	uint32_t NumLumps = LittleLong(header.NumLumps);
+	auto Entries = file->AllocateEntries(NumLumps);
 	header.DirOfs = LittleLong(header.DirOfs);
 	lumps = new RFFLump[header.NumLumps];
-	Reader.Seek (header.DirOfs, FileReader::SeekSet);
+	Reader->Seek (LittleLong(header.DirOfs), FileReader::SeekSet);
-	Reader.Read (lumps, header.NumLumps * sizeof(RFFLump));
+	Reader->Read (lumps, NumLumps * sizeof(RFFLump));
-	BloodCrypt (lumps, header.DirOfs, header.NumLumps * sizeof(RFFLump));
+	BloodCrypt (lumps, LittleLong(header.DirOfs), NumLumps * sizeof(RFFLump));
-
-	Lumps = new FRFFLump[NumLumps];
 
 	for (uint32_t i = 0; i < NumLumps; ++i)
 	{
-		Lumps[i].Position = LittleLong(lumps[i].FilePos);
+		Entries[i].Position = LittleLong(lumps[i].FilePos);
-		Lumps[i].LumpSize = LittleLong(lumps[i].Size);
+		Entries[i].Length = LittleLong(lumps[i].Size);
-		Lumps[i].Owner = this;
+		Entries[i].Flags = 0;
+		Entries[i].Method = METHOD_STORED;
 		if (lumps[i].Flags & 0x10)
 		{
-			Lumps[i].Flags |= LUMPF_COMPRESSED;	// flags the lump as not directly usable
+			Entries[i].Flags = RESFF_COMPRESSED;	// for purposes of decoding, compression and encryption are equivalent.
+			Entries[i].Method = METHOD_RFFCRYPT;
 		}
-		Lumps[i].IndexNum = LittleLong(lumps[i].IndexNum);
+		else
+		{
+			Entries[i].Flags = 0;
+			Entries[i].Method = METHOD_STORED;
+		}
+		Entries[i].Namespace = ns_global;
+		Entries[i].ResourceID = LittleLong(lumps[i].IndexNum);
+	
 		// Rearrange the name and extension to construct the fullname.
 		char name[13];
 		strncpy(name, lumps[i].Name, 8);
@@ -173,66 +137,13 @@ bool FRFFFile::Open(LumpFilterInfo*)
 		name[len+2] = lumps[i].Extension[1];
 		name[len+3] = lumps[i].Extension[2];
 		name[len+4] = 0;
-		Lumps[i].LumpNameSetup(name, stringpool);
+		Entries[i].FileName = file->NormalizeFileName(name);
 	}
 	delete[] lumps;
-	GenerateHash();
+	file->GenerateHash();
 	return true;
 }
 
-FRFFFile::~FRFFFile()
-{
-	if (Lumps != NULL)
-	{
-		delete[] Lumps;
-	}
-}
-
-
-//==========================================================================
-//
-// Get reader (only returns non-NULL if not encrypted)
-//
-//==========================================================================
-
-FileReader *FRFFLump::GetReader()
-{
-	// Don't return the reader if this lump is encrypted
-	// In that case always force caching of the lump
-	if (!(Flags & LUMPF_COMPRESSED))
-	{
-		return FUncompressedLump::GetReader();
-	}
-	else
-	{
-		return NULL;
-	}
-}
-
-//==========================================================================
-//
-// Fills the lump cache and performs decryption
-//
-//==========================================================================
-
-int FRFFLump::FillCache()
-{
-	int res = FUncompressedLump::FillCache();
-
-	if (Flags & LUMPF_COMPRESSED)
-	{
-		int cryptlen = std::min<int> (LumpSize, 256);
-		uint8_t *data = (uint8_t *)Cache;
-
-		for (int i = 0; i < cryptlen; ++i)
-		{
-			data[i] ^= i >> 1;
-		}
-	}
-	return res;
-}
-
-
 //==========================================================================
 //
 // File open
@@ -250,8 +161,8 @@ FResourceFile *CheckRFF(const char *filename, FileReader &file, LumpFilterInfo*
 		file.Seek(0, FileReader::SeekSet);
 		if (!memcmp(head, "RFF\x1a", 4))
 		{
-			auto rf = new FRFFFile(filename, file, sp);
+			auto rf = new FResourceFile(filename, file, sp);
-			if (rf->Open(filter)) return rf;
+			if (OpenRFF(rf, filter)) return rf;
 			file = rf->Destroy();
 		}
 	}

source/common/filesystem/source/file_ssi.cpp

@@ -33,33 +33,9 @@
 **
 */
 
-#include "resourcefile_internal.h"
+#include "resourcefile.h"
 
 namespace FileSys {
-//==========================================================================
-//
-// Build GRP file
-//
-//==========================================================================
-
-class FSSIFile : public FUncompressedFile
-{
-public:
-	FSSIFile(const char * filename, FileReader &file, StringPool* sp);
-	bool Open(int version, int EntryCount, LumpFilterInfo* filter);
-};
-
-
-//==========================================================================
-//
-// Initializes a Build GRP file
-//
-//==========================================================================
-
-FSSIFile::FSSIFile(const char *filename, FileReader &file, StringPool* sp)
-: FUncompressedFile(filename, file, sp)
-{
-}
 
 //==========================================================================
 //
@@ -68,42 +44,50 @@ FSSIFile::FSSIFile(const char *filename, FileReader &file, StringPool* sp)
 //
 //==========================================================================
 
-bool FSSIFile::Open(int version, int EntryCount, LumpFilterInfo*)
+static bool OpenSSI(FResourceFile* file, int version, int EntryCount, LumpFilterInfo*)
 {
-	NumLumps = EntryCount*2;
+	uint32_t NumLumps = EntryCount * 2;
-	Lumps.Resize(EntryCount*2);
+	auto Entries = file->AllocateEntries(NumLumps);
+	auto Reader = file->GetContainerReader();
 	
 
 	int32_t j = (version == 2 ? 267 : 254) + (EntryCount * 121);
 	for (uint32_t i = 0; i < NumLumps; i+=2)
 	{
 		char fn[13];
-		int strlength = Reader.ReadUInt8();
+		int strlength = Reader->ReadUInt8();
 		if (strlength > 12) strlength = 12;
 
-		Reader.Read(fn, 12);
+		Reader->Read(fn, 12);
 		fn[strlength] = 0;
-		int flength = Reader.ReadInt32();
+		int flength = Reader->ReadInt32();
 
-
+		Entries[i].Position = j;
-		Lumps[i].LumpNameSetup(fn, stringpool);
+		Entries[i].Length = flength;
-		Lumps[i].Position = j;
+		Entries[i].Flags = 0;
-		Lumps[i].LumpSize = flength;
+		Entries[i].Namespace = ns_global;
-		Lumps[i].Owner = this;
+		Entries[i].Method = METHOD_STORED;
-		if (strstr(fn, ".GRP")) Lumps[i].Flags |= LUMPF_EMBEDDED;
+		Entries[i].ResourceID = -1;
+		Entries[i].FileName = file->NormalizeFileName(fn);
+		if (strstr(fn, ".GRP")) Entries[i].Flags |= RESFF_EMBEDDED;
 
 		// SSI files can swap the order of the extension's characters - but there's no reliable detection for this and it can be mixed inside the same container, 
 		// so we have no choice but to create another file record for the altered name.
 		std::swap(fn[strlength - 1], fn[strlength - 3]);
-		Lumps[i+1].LumpNameSetup(fn, stringpool);
+
-		Lumps[i+1].Position = j;
+		Entries[i + 1].Position = j;
-		Lumps[i+1].LumpSize = flength;
+		Entries[i + 1].Length = flength;
-		Lumps[i+1].Owner = this;
+		Entries[i + 1].Flags = 0;
-		if (strstr(fn, ".GRP")) Lumps[i+1].Flags |= LUMPF_EMBEDDED;
+		Entries[i + 1].Namespace = ns_global;
+		Entries[i + 1].ResourceID = -1;
+		Entries[i + 1].FileName = file->NormalizeFileName(fn);
+		Entries[i + 1].Method = METHOD_STORED;
+		if (strstr(fn, ".GRP")) Entries[i + 1].Flags |= RESFF_EMBEDDED;
 
 		j += flength;
 
-		Reader.Seek(104, FileReader::SeekCur);
+		Reader->Seek(104, FileReader::SeekCur);
+		file->GenerateHash();
 	}
 	return true;
 }
@@ -145,8 +129,8 @@ FResourceFile* CheckSSI(const char* filename, FileReader& file, LumpFilterInfo*
 			{
 				if (!skipstring(70)) return nullptr;
 			}
-			auto ssi = new FSSIFile(filename, file, sp);
+			auto ssi = new FResourceFile(filename, file, sp);
-			if (ssi->Open(version, numfiles, filter)) return ssi;
+			if (OpenSSI(ssi, version, numfiles, filter)) return ssi;
 			file = ssi->Destroy();
 		}
 	}

source/common/filesystem/source/file_wad.cpp

@@ -37,6 +37,8 @@
 #include "resourcefile.h"
 #include "fs_filesystem.h"
 #include "fs_swap.h"
+#include "fs_stringpool.h"
+#include "resourcefile.h"
 
 namespace FileSys {
 	using namespace byteswap;
@@ -56,71 +58,6 @@ struct wadlump_t
 	char		Name[8];
 };
 
-//==========================================================================
-//
-// Wad Lump (with console doom LZSS support)
-//
-//==========================================================================
-
-class FWadFileLump : public FResourceLump
-{
-public:
-	bool Compressed;
-	int	Position;
-	int Namespace;
-
-	int GetNamespace() const override { return Namespace; }
-
-	int GetFileOffset() override { return Position; }
-	FileReader *GetReader() override
-	{
-		if(!Compressed)
-		{
-			Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
-			return Owner->GetContainerReader();
-		}
-		return NULL;
-	}
-	int FillCache() override
-	{
-		if(!Compressed)
-		{
-			const char * buffer = Owner->GetContainerReader()->GetBuffer();
-
-			if (buffer != NULL)
-			{
-				// This is an in-memory file so the cache can point directly to the file's data.
-				Cache = const_cast<char*>(buffer) + Position;
-				RefCount = -1;
-				return -1;
-			}
-		}
-
-		Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
-		Cache = new char[LumpSize];
-
-		if(Compressed)
-		{
-			FileReader lzss;
-			if (lzss.OpenDecompressor(*Owner->GetContainerReader(), LumpSize, METHOD_LZSS, false, true))
-			{
-				lzss.Read(Cache, LumpSize);
-			}
-		}
-		else
-		{
-			auto read = Owner->GetContainerReader()->Read(Cache, LumpSize);
-			if (read != LumpSize)
-			{
-				throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize);
-			}
-		}
-
-		RefCount = 1;
-		return 1;
-	}
-};
-
 //==========================================================================
 //
 // Wad file
@@ -129,15 +66,12 @@ public:
 
 class FWadFile : public FResourceFile
 {
-	TArray<FWadFileLump> Lumps;
-
 	bool IsMarker(int lump, const char *marker);
 	void SetNamespace(const char *startmarker, const char *endmarker, namespace_t space, FileSystemMessageFunc Printf, bool flathack=false);
 	void SkinHack (FileSystemMessageFunc Printf);
 
 public:
 	FWadFile(const char * filename, FileReader &file, StringPool* sp);
-	FResourceLump *GetLump(int lump) { return &Lumps[lump]; }
 	bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf);
 };
 
@@ -188,56 +122,53 @@ bool FWadFile::Open(LumpFilterInfo*, FileSystemMessageFunc Printf)
 		}
 	}
 
-	TArray<wadlump_t> fileinfo(NumLumps, true);
+	Reader.Seek(InfoTableOfs, FileReader::SeekSet);
-	Reader.Seek (InfoTableOfs, FileReader::SeekSet);
+	auto fd = Reader.Read(NumLumps * sizeof(wadlump_t));
-	Reader.Read (fileinfo.Data(), NumLumps * sizeof(wadlump_t));
+	auto fileinfo = (const wadlump_t*)fd.data();
 
-	Lumps.Resize(NumLumps);
+	AllocateEntries(NumLumps);
 
 	for(uint32_t i = 0; i < NumLumps; i++)
 	{
+		// WAD only supports ASCII. It is also the only format which can use valid backslashes in its names.
 		char n[9];
-		for(int j = 0; j < 8; j++) n[j] = toupper(fileinfo[i].Name[j]);
+		int ishigh = 0;
+		for (int j = 0; j < 8; j++)
+		{
+			if (fileinfo[i].Name[j] & 0x80) ishigh |= 1 << j;
+			n[j] = tolower(fileinfo[i].Name[j]);
+		}
 		n[8] = 0;
-		// This needs to be done differently. We cannot simply assume that all lumps where the first character's high bit is set are compressed without verification.
+		if (ishigh == 1) n[0] &= 0x7f;
-		// This requires explicit toggling for precisely the files that need it.
+		else if (ishigh > 1)
-#if 0
-		Lumps[i].Compressed = !(gameinfo.flags & GI_SHAREWARE) && (n[0] & 0x80) == 0x80;
-		n[0] &= ~0x80;
-#else
-		Lumps[i].Compressed = false;
-#endif
-		Lumps[i].LumpNameSetup(n, stringpool);
-
-		Lumps[i].Owner = this;
-		Lumps[i].Position = isBigEndian ? BigLong(fileinfo[i].FilePos) : LittleLong(fileinfo[i].FilePos);
-		Lumps[i].LumpSize = isBigEndian ? BigLong(fileinfo[i].Size) : LittleLong(fileinfo[i].Size);
-		Lumps[i].Namespace = ns_global;
-		Lumps[i].Flags = Lumps[i].Compressed ? LUMPF_COMPRESSED | LUMPF_SHORTNAME : LUMPF_SHORTNAME;
-
-		// Check if the lump is within the WAD file and print a warning if not.
-		if (Lumps[i].Position + Lumps[i].LumpSize > wadSize || Lumps[i].Position < 0 || Lumps[i].LumpSize < 0)
 		{
-			if (Lumps[i].LumpSize != 0)
+			// This may not end up printing something proper because we do not know what encoding might have been used.
-			{
+			Printf(FSMessageLevel::Warning, "%s: Lump name %.8s contains invalid characters\n", FileName, fileinfo[i].Name);
-				Printf(FSMessageLevel::Warning, "%s: Lump %s contains invalid positioning info and will be ignored\n", FileName, Lumps[i].getName());
-				Lumps[i].clearName();
-			}
-			Lumps[i].LumpSize = Lumps[i].Position = 0;
 		}
+
+		Entries[i].FileName = nullptr;
+		Entries[i].Position = isBigEndian ? BigLong(fileinfo[i].FilePos) : LittleLong(fileinfo[i].FilePos);
+		Entries[i].Length = isBigEndian ? BigLong(fileinfo[i].Size) : LittleLong(fileinfo[i].Size);
+		Entries[i].Namespace = ns_global;
+		Entries[i].Flags = ishigh? RESFF_SHORTNAME | RESFF_COMPRESSED : RESFF_SHORTNAME;
+		Entries[i].Method = ishigh == 1? METHOD_LZSS : METHOD_STORED;
+		Entries[i].FileName = stringpool->Strdup(n);
+		// This doesn't set up the namespace yet.
 	}
 
+
 	GenerateHash(); // Do this before the lump processing below.
 
-	SetNamespace("S_START", "S_END", ns_sprites, Printf);
+	SetNamespace("s_start", "s_end", ns_sprites, Printf);
-	SetNamespace("F_START", "F_END", ns_flats, Printf, true);
+	SetNamespace("f_start", "f_end", ns_flats, Printf, true);
-	SetNamespace("C_START", "C_END", ns_colormaps, Printf);
+	SetNamespace("c_start", "c_end", ns_colormaps, Printf);
-	SetNamespace("A_START", "A_END", ns_acslibrary, Printf);
+	SetNamespace("a_start", "a_end", ns_acslibrary, Printf);
-	SetNamespace("TX_START", "TX_END", ns_newtextures, Printf);
+	SetNamespace("tx_start", "tx_end", ns_newtextures, Printf);
-	SetNamespace("V_START", "V_END", ns_strifevoices, Printf);
+	SetNamespace("v_start", "v_end", ns_strifevoices, Printf);
-	SetNamespace("HI_START", "HI_END", ns_hires, Printf);
+	SetNamespace("hi_start", "hi_end", ns_hires, Printf);
-	SetNamespace("VX_START", "VX_END", ns_voxels, Printf);
+	SetNamespace("vx_start", "vx_end", ns_voxels, Printf);
 	SkinHack(Printf);
+
 	return true;
 }
 
@@ -251,10 +182,10 @@ bool FWadFile::Open(LumpFilterInfo*, FileSystemMessageFunc Printf)
 
 inline bool FWadFile::IsMarker(int lump, const char *marker)
 {
-	if (Lumps[lump].getName()[0] == marker[0])
+	if (Entries[lump].FileName[0] == marker[0])
 	{
-		return (!strcmp(Lumps[lump].getName(), marker) ||
+		return (!strcmp(Entries[lump].FileName, marker) ||
-			(marker[1] == '_' && !strcmp(Lumps[lump].getName() +1, marker)));
+			(marker[1] == '_' && !strcmp(Entries[lump].FileName +1, marker)));
 	}
 	else return false;
 }
@@ -283,20 +214,20 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
 	bool warned = false;
 	int numstartmarkers = 0, numendmarkers = 0;
 	unsigned int i;
-	TArray<Marker> markers;
+	std::vector<Marker> markers;
 
 	for(i = 0; i < NumLumps; i++)
 	{
 		if (IsMarker(i, startmarker))
 		{
 			Marker m = { 0, i };
-			markers.Push(m);
+			markers.push_back(m);
 			numstartmarkers++;
 		}
 		else if (IsMarker(i, endmarker))
 		{
 			Marker m = { 1, i };
-			markers.Push(m);
+			markers.push_back(m);
 			numendmarkers++;
 		}
 	}
@@ -312,15 +243,15 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
 		{
 			// We have found no F_START but one or more F_END markers.
 			// mark all lumps before the last F_END marker as potential flats.
-			unsigned int end = markers[markers.Size()-1].index;
+			unsigned int end = markers[markers.size()-1].index;
 			for(unsigned int ii = 0; ii < end; ii++)
 			{
-				if (Lumps[ii].LumpSize == 4096)
+				if (Entries[ii].Length == 4096)
 				{
 					// We can't add this to the flats namespace but 
 					// it needs to be flagged for the texture manager.
-					Printf(FSMessageLevel::DebugNotify, "%s: Marking %s as potential flat\n", FileName, Lumps[ii].getName());
+					Printf(FSMessageLevel::DebugNotify, "%s: Marking %s as potential flat\n", FileName, Entries[ii].FileName);
-					Lumps[ii].Flags |= LUMPF_MAYBEFLAT;
+					Entries[ii].Flags |= RESFF_MAYBEFLAT;
 				}
 			}
 		}
@@ -328,7 +259,7 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
 	}
 
 	i = 0;
-	while (i < markers.Size())
+	while (i < markers.size())
 	{
 		int start, end;
 		if (markers[i].markertype != 0)
@@ -340,21 +271,21 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
 		start = i++;
 
 		// skip over subsequent x_START markers
-		while (i < markers.Size() && markers[i].markertype == 0)
+		while (i < markers.size() && markers[i].markertype == 0)
 		{
 			Printf(FSMessageLevel::Warning, "%s: duplicate %s marker found.\n", FileName, startmarker);
 			i++;
 			continue;
 		}
 		// same for x_END markers
-		while (i < markers.Size()-1 && (markers[i].markertype == 1 && markers[i+1].markertype == 1))
+		while (i < markers.size()-1 && (markers[i].markertype == 1 && markers[i+1].markertype == 1))
 		{
 			Printf(FSMessageLevel::Warning, "%s: duplicate %s marker found.\n", FileName, endmarker);
 			i++;
 			continue;
 		}
 		// We found a starting marker but no end marker. Ignore this block.
-		if (i >= markers.Size())
+		if (i >= markers.size())
 		{
 			Printf(FSMessageLevel::Warning, "%s: %s marker without corresponding %s found.\n", FileName, startmarker, endmarker);
 			end = NumLumps;
@@ -368,7 +299,7 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
 		Printf(FSMessageLevel::DebugNotify, "%s: Found %s block at (%d-%d)\n", FileName, startmarker, markers[start].index, end);
 		for(int j = markers[start].index + 1; j < end; j++)
 		{
-			if (Lumps[j].Namespace != ns_global)
+			if (Entries[j].Namespace != ns_global)
 			{
 				if (!warned)
 				{
@@ -376,17 +307,17 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
 				}
 				warned = true;
 			}
-			else if (space == ns_sprites && Lumps[j].LumpSize < 8)
+			else if (space == ns_sprites && Entries[j].Length < 8)
 			{
 				// sf 26/10/99:
 				// ignore sprite lumps smaller than 8 bytes (the smallest possible)
 				// in size -- this was used by some dmadds wads
 				// as an 'empty' graphics resource
-				Printf(FSMessageLevel::DebugWarn, "%s: Skipped empty sprite %s (lump %d)\n", FileName, Lumps[j].getName(), j);
+				Printf(FSMessageLevel::DebugWarn, "%s: Skipped empty sprite %s (lump %d)\n", FileName, Entries[j].FileName, j);
 			}
 			else
 			{
-				Lumps[j].Namespace = space;
+				Entries[j].Namespace = space;
 			}
 		}
 	}
@@ -416,11 +347,11 @@ void FWadFile::SkinHack (FileSystemMessageFunc Printf)
 
 	for (i = 0; i < NumLumps; i++)
 	{
-		FResourceLump *lump = &Lumps[i];
+		auto lump = &Entries[i];
 
-		if (!strnicmp(lump->getName(), "S_SKIN", 6))
+		if (!strnicmp(lump->FileName, "S_SKIN", 6))
 		{ // Wad has at least one skin.
-			lump->LumpNameSetup("S_SKIN", nullptr);
+			lump->FileName = "S_SKIN";
 			if (!skinned)
 			{
 				skinned = true;
@@ -428,24 +359,24 @@ void FWadFile::SkinHack (FileSystemMessageFunc Printf)
 
 				for (j = 0; j < NumLumps; j++)
 				{
-					Lumps[j].Namespace = namespc;
+					Entries[j].Namespace = namespc;
 				}
 				namespc++;
 			}
 		}
 		// needless to say, this check is entirely useless these days as map names can be more diverse..
-		if ((lump->getName()[0] == 'M' &&
+		if ((lump->FileName[0] == 'M' &&
-			 lump->getName()[1] == 'A' &&
+			 lump->FileName[1] == 'A' &&
-			 lump->getName()[2] == 'P' &&
+			 lump->FileName[2] == 'P' &&
-			 lump->getName()[3] >= '0' && lump->getName()[3] <= '9' &&
+			 lump->FileName[3] >= '0' && lump->FileName[3] <= '9' &&
-			 lump->getName()[4] >= '0' && lump->getName()[4] <= '9' &&
+			 lump->FileName[4] >= '0' && lump->FileName[4] <= '9' &&
-			 lump->getName()[5] == '\0')
+			 lump->FileName[5] == '\0')
 			||
-			(lump->getName()[0] == 'E' &&
+			(lump->FileName[0] == 'E' &&
-			 lump->getName()[1] >= '0' && lump->getName()[1] <= '9' &&
+			 lump->FileName[1] >= '0' && lump->FileName[1] <= '9' &&
-			 lump->getName()[2] == 'M' &&
+			 lump->FileName[2] == 'M' &&
-			 lump->getName()[3] >= '0' && lump->getName()[3] <= '9' &&
+			 lump->FileName[3] >= '0' && lump->FileName[3] <= '9' &&
-			 lump->getName()[4] == '\0'))
+			 lump->FileName[4] == '\0'))
 		{
 			hasmap = true;
 		}

source/common/filesystem/source/file_whres.cpp

@@ -34,76 +34,61 @@
 **
 */
 
-#include "resourcefile_internal.h"
+#include "resourcefile.h"
 #include "fs_stringpool.h"
 #include "fs_swap.h"
 
 namespace FileSys {
 	using namespace byteswap;
 
-//==========================================================================
-//
-// WH resource file
-//
-//==========================================================================
-
-class FWHResFile : public FUncompressedFile
-{
-	const char* BaseName;
-public:
-	FWHResFile(const char * filename, FileReader &file, StringPool* sp);
-	bool Open(LumpFilterInfo* filter);
-};
-
-
-//==========================================================================
-//
-//
-//
-//==========================================================================
-
-FWHResFile::FWHResFile(const char *filename, FileReader &file, StringPool* sp)
-	: FUncompressedFile(filename, file, sp)
-{
-	BaseName = stringpool->Strdup(ExtractBaseName(filename, false).c_str());
-}
-
 //==========================================================================
 //
 // Open it
 //
 //==========================================================================
 
-bool FWHResFile::Open(LumpFilterInfo*)
+bool OpenWHRes(FResourceFile* file, LumpFilterInfo*)
 {
 	uint32_t directory[1024];
 
-	Reader.Seek(-4096, FileReader::SeekEnd);
+	auto BaseName = ExtractBaseName(file->GetFileName());
-	Reader.Read(directory, 4096);
+	auto Reader = file->GetContainerReader();
+	Reader->Seek(-4096, FileReader::SeekEnd);
+	Reader->Read(directory, 4096);
 
 	int nl =1024/3;
-	Lumps.Resize(nl);
 	
+	int k;
+	for (k = 0; k < nl; k++)
+	{
+		uint32_t offset = LittleLong(directory[k * 3]) * 4096;
+		uint32_t length = LittleLong(directory[k * 3 + 1]);
+		if (length == 0)
+		{
+			break;
+		}
+	}
+	auto Entries = file->AllocateEntries(k);
+	auto NumLumps = k;
 
 	int i = 0;
-	for(int k = 0; k < nl; k++)
+	for(k = 0; k < NumLumps; k++)
 	{
 		uint32_t offset = LittleLong(directory[k*3]) * 4096;
 		uint32_t length = LittleLong(directory[k*3+1]);
-		if (length == 0) break;
 		char num[6];
 		snprintf(num, 6, "/%04d", k);
-		std::string synthname = BaseName;
+		std::string synthname = BaseName + num;
-		synthname += num;
+		
-		Lumps[i].LumpNameSetup(synthname.c_str(), stringpool);
+		Entries[i].Position = offset;
-		Lumps[i].Owner = this;
+		Entries[i].Length = length;
-		Lumps[i].Position = offset;
+		Entries[i].Flags = RESFF_FULLPATH;
-		Lumps[i].LumpSize = length;
+		Entries[i].Namespace = ns_global;
+		Entries[i].ResourceID = -1;
+		Entries[i].Method = METHOD_STORED;
+		Entries[i].FileName = file->NormalizeFileName(synthname.c_str());
 		i++;
 	}
-	NumLumps = i;
-	Lumps.Clamp(NumLumps);
-	Lumps.ShrinkToFit();
 	return true;
 }
 
@@ -134,8 +119,8 @@ FResourceFile *CheckWHRes(const char *filename, FileReader &file, LumpFilterInfo
 			if (offset != checkpos || length == 0 || offset + length >= (size_t)size - 4096 ) return nullptr;
 			checkpos += (length+4095) / 4096;
 		}
-		auto rf = new FWHResFile(filename, file, sp);
+		auto rf = new FResourceFile(filename, file, sp);
-		if (rf->Open(filter)) return rf;
+		if (OpenWHRes(rf, filter)) return rf;
 		file = rf->Destroy();
 	}
 	return NULL;

source/common/filesystem/source/file_zip.cpp

@@ -3,7 +3,7 @@
 **
 **---------------------------------------------------------------------------
 ** Copyright 1998-2009 Randy Heit
-** Copyright 2005-2009 Christoph Oelckers
+** Copyright 2005-2023 Christoph Oelckers
 ** All rights reserved.
 **
 ** Redistribution and use in source and binary forms, with or without
@@ -35,78 +35,18 @@
 
 #include <time.h>
 #include <stdexcept>
-#include "file_zip.h"
 #include "w_zip.h"
 #include "ancientzip.h"
+#include "resourcefile.h"
 #include "fs_findfile.h"
 #include "fs_swap.h"
+#include "fs_stringpool.h"
 
 namespace FileSys {
 	using namespace byteswap;
 
 #define BUFREADCOMMENT (0x400)
 
-//==========================================================================
-//
-// Decompression subroutine
-//
-//==========================================================================
-
-static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdiff_t LumpSize, ptrdiff_t CompressedSize, int GPFlags, bool exceptions)
-{
-	switch (Method)
-	{
-	case METHOD_STORED:
-	{
-		Reader.Read(Cache, LumpSize);
-		break;
-	}
-
-	case METHOD_DEFLATE:
-	case METHOD_BZIP2:
-	case METHOD_LZMA:
-	case METHOD_XZ:
-	{
-		FileReader frz;
-		if (frz.OpenDecompressor(Reader, LumpSize, Method, false, exceptions))
-		{
-			frz.Read(Cache, LumpSize);
-		}
-		break;
-	}
-
-	// Fixme: These should also use a stream
-	case METHOD_IMPLODE:
-	{
-		FZipExploder exploder;
-		if (exploder.Explode((unsigned char*)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize, GPFlags) == -1)
-		{
-			// decompression failed so zero the cache.
-			memset(Cache, 0, LumpSize);
-		}
-		break;
-	}
-
-	case METHOD_SHRINK:
-	{
-		ShrinkLoop((unsigned char *)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize);
-		break;
-	}
-
-	default:
-		assert(0);
-		return false;
-	}
-	return true;
-}
-
-bool FCompressedBuffer::Decompress(char *destbuffer)
-{
-	FileReader mr;
-	mr.OpenMemory(mBuffer, mCompressedSize);
-	return UncompressZipLump(destbuffer, mr, mMethod, mSize, mCompressedSize, mZipFlags, false);
-}
-
 //-----------------------------------------------------------------------
 //
 // Finds the central directory end record in the end of the file.
@@ -166,10 +106,25 @@ static uint32_t Zip_FindCentralDir(FileReader &fin, bool* zip64)
 //
 //==========================================================================
 
+class FZipFile : public FResourceFile
+{
+	void SetEntryAddress(uint32_t entry) override;
+
+public:
+	FZipFile(const char* filename, FileReader& file, StringPool* sp);
+	bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf);
+	FCompressedBuffer GetRawData(uint32_t entry) override;
+};
+
+//==========================================================================
+//
+// Zip file
+//
+//==========================================================================
+
 FZipFile::FZipFile(const char * filename, FileReader &file, StringPool* sp)
 : FResourceFile(filename, file, sp)
 {
-	Lumps = NULL;
 }
 
 bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
@@ -178,8 +133,6 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 	uint32_t centraldir = Zip_FindCentralDir(Reader, &zip64);
 	int skipped = 0;
 
-	Lumps = NULL;
-
 	if (centraldir == 0)
 	{
 		Printf(FSMessageLevel::Error, "%s: ZIP file corrupt!\n", FileName);
@@ -225,15 +178,12 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 		dirsize = info.DirectorySize;
 		DirectoryOffset = info.DirectoryOffset;
 	}
-	Lumps = new FZipLump[NumLumps];
-
 	// Load the entire central directory. Too bad that this contains variable length entries...
 	void *directory = malloc(dirsize);
 	Reader.Seek(DirectoryOffset, FileReader::SeekSet);
 	Reader.Read(directory, dirsize);
 
 	char *dirptr = (char*)directory;
-	FZipLump *lump_p = Lumps;
 
 	std::string name0, name1;
 	bool foundspeciallump = false;
@@ -259,58 +209,11 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 			Printf(FSMessageLevel::Error, "%s: Central directory corrupted.", FileName);
 			return false;
 		}
-
-		for (auto& c : name) c = tolower(c);
-
-		auto vv = name.find("__macosx");
-		if (name.find("filter/") == 0)
-			continue; // 'filter' is a reserved name of the file system.
-		if (name.find("__macosx") == 0) 
-			continue; // skip Apple garbage. At this stage only the root folder matters.
-		if (name.find(".bat") != std::string::npos || name.find(".exe") != std::string::npos)
-			continue; // also ignore executables for this.
-		if (!foundprefix)
-		{
-			// check for special names, if one of these gets found this must be treated as a normal zip.
-			bool isspecial = name.find("/") == std::string::npos ||
-				(filter && std::find(filter->reservedFolders.begin(), filter->reservedFolders.end(), name) != filter->reservedFolders.end());
-			if (isspecial) break;
-			name0 = std::string(name, 0, name.rfind("/")+1);
-			name1 = std::string(name, 0, name.find("/") + 1);
-			foundprefix = true;
 	}
 
-		if (name.find(name0) != 0)
-		{
-			if (!name1.empty())
-			{
-				name0 = name1;
-				if (name.find(name0) != 0)
-				{
-					name0 = "";
-				}
-			}
-			if (name0.empty()) 
-				break;
-		}
-		if (!foundspeciallump && filter)
-		{
-			// at least one of the more common definition lumps must be present.
-			for (auto &p : filter->requiredPrefixes)
-			{ 
-				if (name.find(name0 + p) == 0 || name.rfind(p) == size_t(name.length() - p.length()))
-				{
-					foundspeciallump = true;
-					break;
-				}
-			}
-		}
-	}
-	// If it ran through the list without finding anything it should not attempt any path remapping.
-	if (!foundspeciallump) name0 = "";
-
 	dirptr = (char*)directory;
-	lump_p = Lumps;
+	AllocateEntries(NumLumps);
+	auto Entry = Entries;
 	for (uint32_t i = 0; i < NumLumps; i++)
 	{
 		FZipCentralDirectoryInfo *zip_fh = (FZipCentralDirectoryInfo *)dirptr;
@@ -329,13 +232,6 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 			return false;
 		}
 
-		if (name.find("__macosx") == 0 || name.find("__MACOSX") == 0)
-		{
-			skipped++;
-			continue; // Weed out Apple's resource fork garbage right here because it interferes with safe operation.
-		}
-		if (!name0.empty()) name = std::string(name, name0.length());
-
 		// skip Directories
 		if (name.empty() || (name.back() == '/' && LittleLong(zip_fh->UncompressedSize32) == 0))
 		{
@@ -366,9 +262,6 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 			continue;
 		}
 
-		FixPathSeparator(&name.front());
-		for (auto& c : name) c = tolower(c);
-
 		uint32_t UncompressedSize =LittleLong(zip_fh->UncompressedSize32);
 		uint32_t CompressedSize = LittleLong(zip_fh->CompressedSize32);
 		uint64_t LocalHeaderOffset = LittleLong(zip_fh->LocalHeaderOffset32);
@@ -399,54 +292,59 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 			}
 		}
 
-		lump_p->LumpNameSetup(name.c_str(), stringpool);
+		Entry->FileName = NormalizeFileName(name.c_str());
-		lump_p->LumpSize = UncompressedSize;
+		Entry->Length = UncompressedSize;
-		lump_p->Owner = this;
 		// The start of the Reader will be determined the first time it is accessed.
-		lump_p->Flags = LUMPF_FULLPATH;
+		Entry->Flags = RESFF_FULLPATH | RESFF_NEEDFILESTART;
-		lump_p->NeedFileStart = true;
+		Entry->Method = uint8_t(zip_fh->Method);
-		lump_p->Method = uint8_t(zip_fh->Method);
+		if (Entry->Method != METHOD_STORED) Entry->Flags |= RESFF_COMPRESSED;
-		if (lump_p->Method != METHOD_STORED) lump_p->Flags |= LUMPF_COMPRESSED;
+		if (Entry->Method == METHOD_IMPLODE)
-		lump_p->GPFlags = zip_fh->Flags;
+		{
-		lump_p->CRC32 = zip_fh->CRC32;
+			// for Implode merge the flags into the compression method to make handling in the file system easier and save one variable.
-		lump_p->CompressedSize = CompressedSize;
+			if ((zip_fh->Flags & 6) == 2) Entry->Method = METHOD_IMPLODE_2;
-		lump_p->Position = LocalHeaderOffset;
+			else if ((zip_fh->Flags & 6) == 4) Entry->Method = METHOD_IMPLODE_4;
-		lump_p->CheckEmbedded(filter);
+			else if ((zip_fh->Flags & 6) == 6) Entry->Method = METHOD_IMPLODE_6;
+			else Entry->Method = METHOD_IMPLODE_0;
+		}
+		Entry->CRC32 = zip_fh->CRC32;
+		Entry->CompressedSize = CompressedSize;
+		Entry->Position = LocalHeaderOffset;
+
+		Entry++;
 
-		lump_p++;
 	}
 	// Resize the lump record array to its actual size
 	NumLumps -= skipped;
 	free(directory);
 
 	GenerateHash();
-	PostProcessArchive(&Lumps[0], sizeof(FZipLump), filter);
+	PostProcessArchive(filter);
 	return true;
 }
 
 //==========================================================================
 //
-// Zip file
+//
 //
 //==========================================================================
 
-FZipFile::~FZipFile()
+FCompressedBuffer FZipFile::GetRawData(uint32_t entry)
 {
-	if (Lumps != NULL) delete [] Lumps;
+	FCompressedBuffer cbuf;
-}
 
-//==========================================================================
+	if (entry >= NumLumps >> Entries[entry].Length == 0)
-//
+	{
-//
+		cbuf = { 0, 0, METHOD_STORED, 0, 0, nullptr };
-//
+	}
-//==========================================================================
+	else
+	{
+		auto& e = Entries[entry];
+		cbuf = { e.Length, e.CompressedSize, e.Method, e.CRC32, new char[e.CompressedSize] };
+		if (e.Flags & RESFF_NEEDFILESTART) SetEntryAddress(entry);
+		Reader.Seek(e.Position, FileReader::SeekSet);
+		Reader.Read(cbuf.mBuffer, e.CompressedSize);
+	}
 	
-FCompressedBuffer FZipLump::GetRawData()
-{
-	FCompressedBuffer cbuf = { (unsigned)LumpSize, (unsigned)CompressedSize, Method, GPFlags, CRC32, new char[CompressedSize] };
-	if (NeedFileStart) SetLumpAddress();
-	Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
-	Owner->GetContainerReader()->Read(cbuf.mBuffer, CompressedSize);
 	return cbuf;
 }
 
@@ -456,7 +354,7 @@ FCompressedBuffer FZipLump::GetRawData()
 //
 //==========================================================================
 
-void FZipLump::SetLumpAddress()
+void FZipFile::SetEntryAddress(uint32_t entry)
 {
 	// This file is inside a zip and has not been opened before.
 	// Position points to the start of the local file header, which we must
@@ -464,69 +362,11 @@ void FZipLump::SetLumpAddress()
 	FZipLocalFileHeader localHeader;
 	int skiplen;
 
-	Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
+	Reader.Seek(Entries[entry].Position, FileReader::SeekSet);
-	Owner->GetContainerReader()->Read(&localHeader, sizeof(localHeader));
+	Reader.Read(&localHeader, sizeof(localHeader));
 	skiplen = LittleShort(localHeader.NameLength) + LittleShort(localHeader.ExtraLength);
-	Position += sizeof(localHeader) + skiplen;
+	Entries[entry].Position += sizeof(localHeader) + skiplen;
-	NeedFileStart = false;
+	Entries[entry].Flags &= ~RESFF_NEEDFILESTART;
-}
-
-//==========================================================================
-//
-// Get reader (only returns non-NULL if not encrypted)
-//
-//==========================================================================
-
-FileReader *FZipLump::GetReader()
-{
-	// Don't return the reader if this lump is encrypted
-	// In that case always force caching of the lump
-	if (Method == METHOD_STORED)
-	{
-		if (NeedFileStart) SetLumpAddress();
-		Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
-		return Owner->GetContainerReader();
-	}
-	else return NULL;	
-}
-
-//==========================================================================
-//
-// Fills the lump cache and performs decompression
-//
-//==========================================================================
-
-int FZipLump::FillCache()
-{
-	if (NeedFileStart) SetLumpAddress();
-	const char *buffer;
-
-	if (Method == METHOD_STORED && (buffer = Owner->GetContainerReader()->GetBuffer()) != NULL)
-	{
-		// This is an in-memory file so the cache can point directly to the file's data.
-		Cache = const_cast<char*>(buffer) + Position;
-		RefCount = -1;
-		return -1;
-	}
-
-	Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
-	Cache = new char[LumpSize];
-	UncompressZipLump(Cache, *Owner->GetContainerReader(), Method, LumpSize, CompressedSize, GPFlags, true);
-	RefCount = 1;
-	return 1;
-}
-
-//==========================================================================
-//
-//
-//
-//==========================================================================
-
-int FZipLump::GetFileOffset()
-{
-	if (Method != METHOD_STORED) return -1;
-	if (NeedFileStart) SetLumpAddress();
-	return (int)Position;
 }
 
 //==========================================================================
@@ -555,171 +395,4 @@ FResourceFile *CheckZip(const char *filename, FileReader &file, LumpFilterInfo*
 }
 
 
-
-//==========================================================================
-//
-// time_to_dos
-//
-// Converts time from struct tm to the DOS format used by zip files.
-//
-//==========================================================================
-
-static std::pair<uint16_t, uint16_t> time_to_dos(struct tm *time)
-{
-	std::pair<uint16_t, uint16_t> val;
-	if (time == NULL || time->tm_year < 80)
-	{
-		val.first = val.second = 0;
-	}
-	else
-	{
-		val.first = (time->tm_year - 80) * 512 + (time->tm_mon + 1) * 32 + time->tm_mday;
-		val.second= time->tm_hour * 2048 + time->tm_min * 32 + time->tm_sec / 2;
-	}
-	return val;
-}
-
-//==========================================================================
-//
-// append_to_zip
-//
-// Write a given file to the zipFile.
-// 
-// zipfile: zip object to be written to
-// 
-// returns: position = success, -1 = error
-//
-//==========================================================================
-
-static int AppendToZip(FileWriter *zip_file, const FCompressedBuffer &content, std::pair<uint16_t, uint16_t> &dostime)
-{
-	FZipLocalFileHeader local;
-	int position;
-
-	local.Magic = ZIP_LOCALFILE;
-	local.VersionToExtract[0] = 20;
-	local.VersionToExtract[1] = 0;
-	local.Flags = content.mMethod == METHOD_DEFLATE ? LittleShort((uint16_t)2) : LittleShort((uint16_t)content.mZipFlags);
-	local.Method = LittleShort((uint16_t)content.mMethod);
-	local.ModDate = LittleShort(dostime.first);
-	local.ModTime = LittleShort(dostime.second);
-	local.CRC32 = content.mCRC32;
-	local.UncompressedSize = LittleLong((unsigned)content.mSize);
-	local.CompressedSize = LittleLong((unsigned)content.mCompressedSize);
-	local.NameLength = LittleShort((unsigned short)strlen(content.filename));
-	local.ExtraLength = 0;
-
-	// Fill in local directory header.
-
-	position = (int)zip_file->Tell();
-
-	// Write out the header, file name, and file data.
-	if (zip_file->Write(&local, sizeof(local)) != sizeof(local) ||
-		zip_file->Write(content.filename, strlen(content.filename)) != strlen(content.filename) ||
-		zip_file->Write(content.mBuffer, content.mCompressedSize) != content.mCompressedSize)
-	{
-		return -1;
-	}
-	return position;
-}
-
-
-//==========================================================================
-//
-// write_central_dir
-//
-// Writes the central directory entry for a file.
-//
-//==========================================================================
-
-int AppendCentralDirectory(FileWriter *zip_file, const FCompressedBuffer &content, std::pair<uint16_t, uint16_t> &dostime, int position)
-{
-	FZipCentralDirectoryInfo dir;
-
-	dir.Magic = ZIP_CENTRALFILE;
-	dir.VersionMadeBy[0] = 20;
-	dir.VersionMadeBy[1] = 0;
-	dir.VersionToExtract[0] = 20;
-	dir.VersionToExtract[1] = 0;
-	dir.Flags = content.mMethod == METHOD_DEFLATE ? LittleShort((uint16_t)2) : LittleShort((uint16_t)content.mZipFlags);
-	dir.Method = LittleShort((uint16_t)content.mMethod);
-	dir.ModTime = LittleShort(dostime.first);
-	dir.ModDate = LittleShort(dostime.second);
-	dir.CRC32 = content.mCRC32;
-	dir.CompressedSize32 = LittleLong((unsigned)content.mCompressedSize);
-	dir.UncompressedSize32 = LittleLong((unsigned)content.mSize);
-	dir.NameLength = LittleShort((unsigned short)strlen(content.filename));
-	dir.ExtraLength = 0;
-	dir.CommentLength = 0;
-	dir.StartingDiskNumber = 0;
-	dir.InternalAttributes = 0;
-	dir.ExternalAttributes = 0;
-	dir.LocalHeaderOffset32 = LittleLong((unsigned)position);
-
-	if (zip_file->Write(&dir, sizeof(dir)) != sizeof(dir) ||
-		zip_file->Write(content.filename,  strlen(content.filename)) != strlen(content.filename))
-	{
-		return -1;
-	}
-	return 0;
-}
-
-bool WriteZip(const char* filename, const FCompressedBuffer* content, size_t contentcount)
-{
-	// try to determine local time
-	struct tm *ltime;
-	time_t ttime;
-	ttime = time(nullptr);
-	ltime = localtime(&ttime);
-	auto dostime = time_to_dos(ltime);
-
-	TArray<int> positions;
-
-	auto f = FileWriter::Open(filename);
-	if (f != nullptr)
-	{
-		for (size_t i = 0; i < contentcount; i++)
-		{
-			int pos = AppendToZip(f, content[i], dostime);
-			if (pos == -1)
-			{
-				delete f;
-				remove(filename);
-				return false;
-			}
-			positions.Push(pos);
-		}
-
-		int dirofs = (int)f->Tell();
-		for (size_t i = 0; i < contentcount; i++)
-		{
-			if (AppendCentralDirectory(f, content[i], dostime, positions[i]) < 0)
-			{
-				delete f;
-				remove(filename);
-				return false;
-			}
-		}
-
-		// Write the directory terminator.
-		FZipEndOfCentralDirectory dirend;
-		dirend.Magic = ZIP_ENDOFDIR;
-		dirend.DiskNumber = 0;
-		dirend.FirstDisk = 0;
-		dirend.NumEntriesOnAllDisks = dirend.NumEntries = LittleShort((uint16_t)contentcount);
-		dirend.DirectoryOffset = LittleLong((unsigned)dirofs);
-		dirend.DirectorySize = LittleLong((uint32_t)(f->Tell() - dirofs));
-		dirend.ZipCommentLength = 0;
-		if (f->Write(&dirend, sizeof(dirend)) != sizeof(dirend))
-		{
-			delete f;
-			remove(filename);
-			return false;
-		}
-		delete f;
-		return true;
-	}
-	return false;
-}
-
 }

source/common/filesystem/source/file_zip.h

@@ -1,51 +0,0 @@
-#ifndef __FILE_ZIP_H
-#define __FILE_ZIP_H
-
-#include "resourcefile.h"
-
-namespace FileSys {
-//==========================================================================
-//
-// Zip Lump
-//
-//==========================================================================
-
-struct FZipLump : public FResourceLump
-{
-	uint16_t	GPFlags;
-	uint8_t	Method;
-	bool	NeedFileStart;
-	int		CompressedSize;
-	int64_t		Position;
-	unsigned CRC32;
-
-	virtual FileReader *GetReader();
-	virtual int FillCache() override;
-
-private:
-	void SetLumpAddress();
-	virtual int GetFileOffset();
-	FCompressedBuffer GetRawData();
-};
-
-
-//==========================================================================
-//
-// Zip file
-//
-//==========================================================================
-
-class FZipFile : public FResourceFile
-{
-	FZipLump *Lumps;
-
-public:
-	FZipFile(const char * filename, FileReader &file, StringPool* sp);
-	virtual ~FZipFile();
-	bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf);
-	virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; }
-};
-
-}
-
-#endif

source/common/filesystem/source/files.cpp

@@ -35,6 +35,9 @@
 
 #include <string>
 #include <memory>
+#include <algorithm>
+#include <assert.h>
+#include <string.h>
 #include "files_internal.h"
 
 namespace FileSys {
@@ -326,15 +329,15 @@ char *MemoryReader::Gets(char *strbuf, ptrdiff_t len)
 int BufferingReader::FillBuffer(ptrdiff_t newpos)
 {
 	if (newpos > Length) newpos = Length;
-	if (newpos < bufferpos) return 0;
+	if (newpos <= bufferpos) return 0;
-	auto read = baseReader->Read(&buf[bufferpos], newpos - bufferpos);
+	auto read = baseReader->Read(&buf.writable()[bufferpos], newpos - bufferpos);
 	bufferpos += read;
 	if (bufferpos == Length)
 	{
 		// we have read the entire file, so delete our data provider.
 		baseReader.reset();
 	}
-	return read == newpos - bufferpos ? 0 : -1;
+	return newpos == bufferpos ? 0 : -1;
 }
 
 ptrdiff_t BufferingReader::Seek(ptrdiff_t offset, int origin)
@@ -392,45 +395,40 @@ bool FileReader::OpenMemory(const void *mem, FileReader::Size length)
 	return true;
 }
 
-bool FileReader::OpenMemoryArray(const void *mem, FileReader::Size length)
+bool FileReader::OpenMemoryArray(FileData& data)
 {
 	Close();
-	mReader = new MemoryArrayReader<std::vector<uint8_t>>((const char *)mem, length);
+	if (data.size() > 0) mReader = new MemoryArrayReader(data);
 	return true;
 }
 
-bool FileReader::OpenMemoryArray(std::vector<uint8_t>& data)
+FileData FileReader::Read(size_t len)
 {
-	Close();
+	FileData buffer;
-	if (data.size() > 0) mReader = new MemoryArrayReader<std::vector<uint8_t>>(data);
+	if (len > 0)
-	return true;
-}
-
-bool FileReader::OpenMemoryArray(ResourceData& data)
-{
-	Close();
-	if (data.size() > 0) mReader = new MemoryArrayReader<ResourceData>(data);
-	return true;
-}
-
-bool FileReader::OpenMemoryArray(std::function<bool(std::vector<uint8_t>&)> getter)
-{
-	auto reader = new MemoryArrayReader<std::vector<uint8_t>>(nullptr, 0);
-	if (getter(reader->GetArray()))
 	{
-		Close();
+		Size length = mReader->Read(buffer.allocate(len), len);
-		reader->UpdateBuffer();
+		if ((size_t)length < len) buffer.allocate(length);
-		mReader = reader;
-		return true;
-	}
-	else
-	{
-		// This will keep the old buffer, if one existed
-		delete reader;
-		return false;
 	}
+	return buffer;
 }
 
+FileData FileReader::ReadPadded(size_t padding)
+{
+	auto len = GetLength();
+	FileData buffer;
+
+	if (len > 0)
+	{
+		auto p = (char*)buffer.allocate(len + padding);
+		Size length = mReader->Read(p, len);
+		if (length < len) buffer.clear();
+		else memset(p + len, 0, padding);
+	}
+	return buffer;
+}
+
+
 
 //==========================================================================
 //

source/common/filesystem/source/files_decompress.cpp

@@ -45,6 +45,9 @@
 #include <stdexcept>
 
 #include "fs_files.h"
+#include "files_internal.h"
+#include "ancientzip.h"
+#include "fs_decompress.h"
 
 namespace FileSys {
 	using namespace byteswap;
@@ -53,6 +56,7 @@ namespace FileSys {
 class DecompressorBase : public FileReaderInterface
 {
 	bool exceptions = false;
+
 public:
 	// These do not work but need to be defined to satisfy the FileReaderInterface.
 	// They will just error out when called.
@@ -64,6 +68,10 @@ public:
 	void EnableExceptions(bool on) { exceptions = on; }
 
 protected:
+	DecompressorBase()
+	{
+		//seekable = false;
+	}
 	FileReader* File = nullptr;
 	FileReader OwnedFile;
 };
@@ -174,7 +182,7 @@ public:
 		inflateEnd (&Stream);
 	}
 
-	ptrdiff_t Read (void *buffer, ptrdiff_t len) override
+	ptrdiff_t Read (void *buffer, ptrdiff_t olen) override
 	{
 		int err = 0;
 
@@ -183,6 +191,7 @@ public:
 			DecompressionError("File not open");
 			return 0;
 		}
+		auto len = olen;
 		if (len == 0) return 0;
 
 		while (len > 0)
@@ -215,7 +224,7 @@ public:
 			return 0;
 		}
 
-		return len - Stream.avail_out;
+		return olen - Stream.avail_out;
 	}
 
 	void FillBuffer ()
@@ -841,19 +850,22 @@ public:
 };
 
 
-bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions)
+bool OpenDecompressor(FileReader& self, FileReader &parent, FileReader::Size length, int method, int flags)
 {
+	FileReaderInterface* fr = nullptr;
 	DecompressorBase* dec = nullptr;
 	try
 	{
 		FileReader* p = &parent;
-		switch (method & ~METHOD_TRANSFEROWNER)
+		bool exceptions = !!(flags & DCF_EXCEPTIONS);
+
+		switch (method)
 		{
 		case METHOD_DEFLATE:
 		case METHOD_ZLIB:
 		{
 			auto idec = new DecompressorZ;
-			dec = idec;
+			fr = dec = idec;
 			idec->EnableExceptions(exceptions);
 			if (!idec->Open(p, method == METHOD_DEFLATE))
 			{
@@ -865,7 +877,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 		case METHOD_BZIP2:
 		{
 			auto idec = new DecompressorBZ2;
-			dec = idec;
+			fr = dec = idec;
 			idec->EnableExceptions(exceptions);
 			if (!idec->Open(p))
 			{
@@ -877,7 +889,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 		case METHOD_LZMA:
 		{
 			auto idec = new DecompressorLZMA;
-			dec = idec;
+			fr = dec = idec;
 			idec->EnableExceptions(exceptions);
 			if (!idec->Open(p, length))
 			{
@@ -889,7 +901,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 		case METHOD_XZ:
 		{
 			auto idec = new DecompressorXZ;
-			dec = idec;
+			fr = dec = idec;
 			idec->EnableExceptions(exceptions);
 			if (!idec->Open(p, length))
 			{
@@ -901,7 +913,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 		case METHOD_LZSS:
 		{
 			auto idec = new DecompressorLZSS;
-			dec = idec;
+			fr = dec = idec;
 			idec->EnableExceptions(exceptions);
 			if (!idec->Open(p))
 			{
@@ -911,34 +923,103 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 			break;
 		}
 
-		// todo: METHOD_IMPLODE, METHOD_SHRINK
+		// The decoders for these legacy formats can only handle the full data in one go so we have to perform the entire decompression here.
+		case METHOD_IMPLODE_0:
+		case METHOD_IMPLODE_2:
+		case METHOD_IMPLODE_4:
+		case METHOD_IMPLODE_6:
+		{
+			FileData buffer(nullptr, length);
+			FZipExploder exploder;
+			if (exploder.Explode(buffer.writable(), length, *p, p->GetLength(), method - METHOD_IMPLODE_MIN) == -1)
+			{
+				if (exceptions)
+				{
+					throw FileSystemException("DecompressImplode failed");
+				}
+				return false;
+			}
+			fr = new MemoryArrayReader(buffer);
+			flags &= ~(DCF_SEEKABLE | DCF_CACHED);
+			break;
+		}
+
+		case METHOD_SHRINK:
+		{
+			FileData buffer(nullptr, length);
+			ShrinkLoop(buffer.writable(), length, *p, p->GetLength()); // this never fails.
+			fr = new MemoryArrayReader(buffer);
+			flags &= ~(DCF_SEEKABLE | DCF_CACHED);
+			break;
+		}
+
+		// While this could be made a buffering reader it isn't worth the effort because only stock RFFs are encrypted and they do not contain large files.
+		case METHOD_RFFCRYPT:
+		{
+			FileData buffer = p->Read(length);
+			auto bufr = buffer.writable();
+			FileReader::Size cryptlen = std::min<FileReader::Size>(length, 256);
+
+			for (FileReader::Size i = 0; i < cryptlen; ++i)
+			{
+				bufr[i] ^= i >> 1;
+			}
+			fr = new MemoryArrayReader(buffer);
+			flags &= ~(DCF_SEEKABLE | DCF_CACHED);
+			break;
+		}
+
 		default:
 			return false;
 		}
-		if (method & METHOD_TRANSFEROWNER)
+		if (dec)
+		{
+			if (flags & DCF_TRANSFEROWNER)
 			{
 				dec->SetOwnsReader();
 			}
-
 			dec->Length = length;
-		if (!seekable)
+		}
+		if ((flags & DCF_CACHED))
 		{
-			Close();
+			// read everything into a MemoryArrayReader.
-			mReader = dec;
+			FileData data(nullptr, length);
+			fr->Read(data.writable(), length);
+			fr = new MemoryArrayReader(data);
+		}
+		else if ((flags & DCF_SEEKABLE))
+		{
+			// create a wrapper that can buffer the content so that seeking is possible
+			fr = new BufferingReader(fr);
+		}
+		self = FileReader(fr);
 		return true;
 			}
-		else
-		{
-			// todo: create a wrapper. for now this fails
-			delete dec;
-			return false;
-		}
-	}
 	catch (...)
 	{
-		if (dec) delete dec;
+		if (fr) delete fr;
 		throw;
 	}
 }
 
+
+bool FCompressedBuffer::Decompress(char* destbuffer)
+{
+	if (mMethod == METHOD_STORED)
+	{
+		memcpy(destbuffer, mBuffer, mSize);
+		return true;
+	}
+	else
+	{
+		FileReader mr;
+		mr.OpenMemory(mBuffer, mCompressedSize);
+		FileReader frz;
+		if (OpenDecompressor(frz, mr, mSize, mMethod))
+		{
+			return frz.Read(destbuffer, mSize) != mSize;
+		}
+	}
+	return false;
+}
 }

source/common/filesystem/source/files_internal.h

@@ -31,7 +31,7 @@ public:
 
 class BufferingReader : public MemoryReader
 {
-	std::vector<uint8_t> buf;
+	FileData buf;
 	std::unique_ptr<FileReaderInterface> baseReader;
 	ptrdiff_t bufferpos = 0;
 
@@ -40,6 +40,9 @@ public:
 	BufferingReader(FileReaderInterface* base)
 		: baseReader(base)
 	{
+		Length = base->Length;
+		buf.allocate(Length);
+		bufptr = (const char*)buf.data();
 	}
 
 	ptrdiff_t Seek(ptrdiff_t offset, int origin) override;
@@ -55,10 +58,9 @@ public:
 //
 //==========================================================================
 
-template<class T>
 class MemoryArrayReader : public MemoryReader
 {
-	T buf;
+	FileData buf;
 
 public:
 	MemoryArrayReader()
@@ -67,24 +69,18 @@ public:
 		Length = 0;
 	}
 
-	MemoryArrayReader(const char* buffer, ptrdiff_t length)
+	MemoryArrayReader(size_t len)
 	{
-		if (length > 0)
+		buf.allocate(len);
-		{
-			buf.resize(length);
-			memcpy(&buf[0], buffer, length);
-		}
 		UpdateBuffer();
 	}
 
-	MemoryArrayReader(T& buffer)
+	MemoryArrayReader(FileData& buffer)
 	{
 		buf = std::move(buffer);
 		UpdateBuffer();
 	}
 
-	std::vector<uint8_t>& GetArray() { return buf; }
-
 	void UpdateBuffer()
 	{
 		bufptr = (const char*)buf.data();
@@ -93,7 +89,4 @@ public:
 	}
 };
 
-bool OpenMemoryArray(std::vector<uint8_t>& data);	// take the given array
-
-
 }

source/common/filesystem/source/filesystem.cpp

@@ -41,7 +41,7 @@
 #include <ctype.h>
 #include <string.h>
 
-#include "resourcefile_internal.h"
+#include "resourcefile.h"
 #include "fs_filesystem.h"
 #include "fs_findfile.h"
 #include "md5.hpp"
@@ -90,29 +90,37 @@ static void md5Hash(FileReader& reader, uint8_t* digest)
 
 struct FileSystem::LumpRecord
 {
-	FResourceLump *lump;
+	FResourceFile *resfile;
 	LumpShortName shortName;
 	const char*	LongName;
-	int			rfnum;
+	int			resindex;
-	int			Namespace;
+	int16_t		rfnum;		// this is not necessarily the same as resfile's index!
+	int16_t		Namespace;
 	int			resourceId;
 	int			flags;
 
-	void SetFromLump(int filenum, FResourceLump* lmp, StringPool* sp)
+	void SetFromLump(FResourceFile* file, int fileindex, int filenum, StringPool* sp, const char* name = nullptr)
 	{
-		lump = lmp;
+		if (fileindex == 649 && filenum == 0)
+		{
+			int a = 0;
+		}
+		resfile = file;
+		resindex = fileindex;
 		rfnum = filenum;
 		flags = 0;
 
-		if (lump->Flags & LUMPF_SHORTNAME)
+		auto lflags = file->GetEntryFlags(fileindex);
+		if (!name) name = file->getName(fileindex);
+		if (lflags & RESFF_SHORTNAME)
 		{
-			UpperCopy(shortName.String, lump->getName());
+			UpperCopy(shortName.String, name);
 			shortName.String[8] = 0;
 			LongName = "";
-			Namespace = lump->GetNamespace();
+			Namespace = file->GetEntryNamespace(fileindex);
 			resourceId = -1;
 		}
-		else if ((lump->Flags & LUMPF_EMBEDDED) || !lump->getName() || !*lump->getName())
+		else if ((lflags & RESFF_EMBEDDED) || !name || !*name)
 		{
 			shortName.qword = 0;
 			LongName = "";
@@ -121,8 +129,8 @@ struct FileSystem::LumpRecord
 		}
 		else
 		{
-			LongName = lump->getName();
+			LongName = name;
-			resourceId = lump->GetIndexNum();
+			resourceId = file->GetEntryResourceID(fileindex);
 
 			// Map some directories to WAD namespaces.
 			// Note that some of these namespaces don't exist in WADS.
@@ -206,11 +214,6 @@ void FileSystem::DeleteAll ()
 	Hashes.clear();
 	NumEntries = 0;
 
-	// explicitly delete all manually added lumps.
-	for (auto &frec : FileInfo)
-	{
-		if (frec.rfnum == -1) delete frec.lump;
-	}
 	FileInfo.clear();
 	for (int i = (int)Files.size() - 1; i >= 0; --i)
 	{
@@ -242,6 +245,8 @@ bool FileSystem::InitMultipleFiles (std::vector<std::string>& filenames, LumpFil
 {
 	int numfiles;
 
+	// the first call here will designate a main thread which may use shared file readers. All other thewads have to open new file handles.
+	SetMainThread();
 	// open all the files, load headers, and count lumps
 	DeleteAll();
 	numfiles = 0;
@@ -295,23 +300,23 @@ bool FileSystem::InitMultipleFiles (std::vector<std::string>& filenames, LumpFil
 //
 //==========================================================================
 
-FResourceFile* CheckLump(const char* filename, FileReader& file, LumpFilterInfo*, FileSystemMessageFunc Printf, StringPool* sp);
-
 int FileSystem::AddFromBuffer(const char* name, char* data, int size, int id, int flags)
 {
 	FileReader fr;
-	fr.OpenMemoryArray((uint8_t*)data, size);
+	FileData blob(data, size);
+	fr.OpenMemoryArray(blob);
+
+	// wrap this into a single lump resource file (should be done a little better later.)
+	auto rf = new FResourceFile(name, fr, stringpool);
+	auto Entries = rf->AllocateEntries(1);
+	Entries[0].FileName = rf->NormalizeFileName(ExtractBaseName(name, true).c_str());
+	Entries[0].ResourceID = -1;
+	Entries[0].Length = size;
 
-	// just wrap this into a single lump resource file (should be done a little better later.)
-	auto rf = CheckLump(name, fr, nullptr, nullptr, stringpool);
-	if (rf)
-	{
 	Files.push_back(rf);
-		FResourceLump* lump = rf->GetLump(0);
 	FileInfo.resize(FileInfo.size() + 1);
 	FileSystem::LumpRecord* lump_p = &FileInfo.back();
-		lump_p->SetFromLump((int)Files.size(), lump, stringpool);
+	lump_p->SetFromLump(rf, 0, (int)Files.size() - 1, stringpool);
-	}
 	return (int)FileInfo.size() - 1;
 }
 
@@ -377,25 +382,23 @@ void FileSystem::AddFile (const char *filename, FileReader *filer, LumpFilterInf
 		uint32_t lumpstart = (uint32_t)FileInfo.size();
 
 		resfile->SetFirstLump(lumpstart);
+		Files.push_back(resfile);
 		for (int i = 0; i < resfile->EntryCount(); i++)
 		{
-			FResourceLump* lump = resfile->GetLump(i);
 			FileInfo.resize(FileInfo.size() + 1);
 			FileSystem::LumpRecord* lump_p = &FileInfo.back();
-			lump_p->SetFromLump((int)Files.size(), lump, stringpool);
+			lump_p->SetFromLump(resfile, i, (int)Files.size() - 1, stringpool);
 		}
 
-		Files.push_back(resfile);
-
 		for (int i = 0; i < resfile->EntryCount(); i++)
 		{
 			int flags = resfile->GetEntryFlags(i);
-			if (flags & LUMPF_EMBEDDED)
+			if (flags & RESFF_EMBEDDED)
 			{
 				std::string path = filename;
 				path += ':';
 				path += resfile->getName(i);
-				auto embedded = resfile->GetEntryReader(i, true);
+				auto embedded = resfile->GetEntryReader(i, READER_NEW, READERFLAG_SEEKABLE);
 				AddFile(path.c_str(), &embedded, filter, Printf, hashfile);
 			}
 		}
@@ -425,9 +428,9 @@ void FileSystem::AddFile (const char *filename, FileReader *filer, LumpFilterInf
 			for (int i = 0; i < resfile->EntryCount(); i++)
 			{
 				int flags = resfile->GetEntryFlags(i);
-				if (!(flags & LUMPF_EMBEDDED))
+				if (!(flags & RESFF_EMBEDDED))
 				{
-					auto reader = resfile->GetEntryReader(i, true);
+					auto reader = resfile->GetEntryReader(i, READER_SHARED, 0);
 					md5Hash(filereader, cksum);
 
 					for (size_t j = 0; j < sizeof(cksum); ++j)
@@ -529,8 +532,9 @@ int FileSystem::CheckNumForName (const char *name, int space) const
 			// If we find a lump with this name in the global namespace that does not come
 			// from a Zip return that. WADs don't know these namespaces and single lumps must
 			// work as well.
+			auto lflags = lump.resfile->GetEntryFlags(lump.resindex);
 			if (space > ns_specialzipdirectory && lump.Namespace == ns_global && 
-				!((lump.lump->Flags ^lump.flags) & LUMPF_FULLPATH)) break;
+				!((lflags ^lump.flags) & RESFF_FULLPATH)) break;
 		}
 		i = NextLumpIndex[i];
 	}
@@ -770,31 +774,14 @@ int FileSystem::GetResource (int resid, const char *type, int filenum) const
 //
 //==========================================================================
 
-int FileSystem::FileLength (int lump) const
+ptrdiff_t FileSystem::FileLength (int lump) const
 {
 	if ((size_t)lump >= NumEntries)
 	{
 		return -1;
 	}
-	return FileInfo[lump].lump->LumpSize;
+	const auto &lump_p = FileInfo[lump];
-}
+	return (int)lump_p.resfile->Length(lump_p.resindex);
-
-//==========================================================================
-//
-// GetFileOffset
-//
-// Returns the offset from the beginning of the file to the lump.
-// Returns -1 if the lump is compressed or can't be read directly
-//
-//==========================================================================
-
-int FileSystem::GetFileOffset (int lump)
-{
-	if ((size_t)lump >= NumEntries)
-	{
-		return -1;
-	}
-	return FileInfo[lump].lump->GetFileOffset();
 }
 
 //==========================================================================
@@ -810,7 +797,8 @@ int FileSystem::GetFileFlags (int lump)
 		return 0;
 	}
 
-	return FileInfo[lump].lump->Flags ^ FileInfo[lump].flags;
+	const auto& lump_p = FileInfo[lump];
+	return lump_p.resfile->GetEntryFlags(lump_p.resindex) ^ lump_p.flags;
 }
 
 //==========================================================================
@@ -906,8 +894,6 @@ void FileSystem::RenameFile(int num, const char* newfn)
 //
 //==========================================================================
 
-static FResourceLump placeholderLump;
-
 void FileSystem::MoveLumpsInFolder(const char *path)
 {
 	if (FileInfo.size() == 0)
@@ -925,11 +911,12 @@ void FileSystem::MoveLumpsInFolder(const char *path)
 		if (li.rfnum >= GetIwadNum()) break;
 		if (strnicmp(li.LongName, path, len) == 0)
 		{
-			FileInfo.push_back(li);
+			auto lic = li;	// make a copy before pushing.
-			li.lump = &placeholderLump;			// Make the old entry point to something empty. We cannot delete the lump record here because it'd require adjustment of all indices in the list.
+			FileInfo.push_back(lic);
+			li.LongName = "";	//nuke the name of the old record.
+			li.shortName.qword = 0;
 			auto &ln = FileInfo.back();
-			ln.lump->LumpNameSetup(ln.LongName + len, stringpool); // may be able to avoid the string allocation!
+			ln.SetFromLump(li.resfile, li.resindex, rfnum, stringpool, ln.LongName + len);
-			ln.SetFromLump(rfnum, ln.lump, stringpool);
 		}
 	}
 }
@@ -1309,9 +1296,7 @@ FileData FileSystem::ReadFile (int lump)
 	{
 		throw FileSystemException("ReadFile: %u >= NumEntries", lump);
 	}
-	auto lumpp = FileInfo[lump].lump;
+	return FileInfo[lump].resfile->Read(FileInfo[lump].resindex);
-
-	return FileData(lumpp);
 }
 
 //==========================================================================
@@ -1323,53 +1308,31 @@ FileData FileSystem::ReadFile (int lump)
 //==========================================================================
 
 
-FileReader FileSystem::OpenFileReader(int lump)
+FileReader FileSystem::OpenFileReader(int lump, int readertype, int readerflags)
 {
 	if ((unsigned)lump >= (unsigned)FileInfo.size())
 	{
 		throw FileSystemException("OpenFileReader: %u >= NumEntries", lump);
 	}
 
-	auto rl = FileInfo[lump].lump;
+	auto file = FileInfo[lump].resfile;
-	auto rd = rl->GetReader();
+	return file->GetEntryReader(FileInfo[lump].resindex, readertype, readerflags);
-
-	if (rl->RefCount == 0 && rd != nullptr && !rd->GetBuffer() && !(rl->Flags & LUMPF_COMPRESSED))
-	{
-		FileReader rdr;
-		rdr.OpenFilePart(*rd, rl->GetFileOffset(), rl->LumpSize);
-		return rdr;
-	}
-	return rl->NewReader();	// This always gets a reader to the cache
-}
-
-FileReader FileSystem::ReopenFileReader(int lump, bool alwayscache)
-{
-	if ((unsigned)lump >= (unsigned)FileInfo.size())
-	{
-		throw FileSystemException("ReopenFileReader: %u >= NumEntries", lump);
-	}
-
-	auto rl = FileInfo[lump].lump;
-	auto rd = rl->GetReader();
-
-	if (rl->RefCount == 0 && rd != nullptr && !rd->GetBuffer() && !alwayscache && !(rl->Flags & LUMPF_COMPRESSED))
-	{
-		int fileno = GetFileContainer(lump);
-		const char *filename = GetResourceFileFullName(fileno);
-		FileReader fr;
-		if (fr.OpenFile(filename, rl->GetFileOffset(), rl->LumpSize))
-		{
-			return fr;
-		}
-	}
-	return rl->NewReader();	// This always gets a reader to the cache
 }
 
 FileReader FileSystem::OpenFileReader(const char* name)
 {
+	FileReader fr;
 	auto lump = CheckNumForFullName(name);
-	if (lump < 0) return FileReader();
+	if (lump >= 0) fr = OpenFileReader(lump);
-	else return OpenFileReader(lump);
+	return fr;
+}
+
+FileReader FileSystem::ReopenFileReader(const char* name, bool alwayscache)
+{
+	FileReader fr;
+	auto lump = CheckNumForFullName(name);
+	if (lump >= 0) fr = ReopenFileReader(lump, alwayscache);
+	return fr;
 }
 
 //==========================================================================
@@ -1498,7 +1461,7 @@ bool FileSystem::CreatePathlessCopy(const char *name, int id, int /*flags*/)
 
 	if (slash == nullptr)
 	{
-		FileInfo[lump].flags = LUMPF_FULLPATH;
+		FileInfo[lump].flags = RESFF_FULLPATH;
 		return true;	// already is pathless.
 	}
 
@@ -1506,7 +1469,7 @@ bool FileSystem::CreatePathlessCopy(const char *name, int id, int /*flags*/)
 	// just create a new reference to the original data with a different name.
 	oldlump.LongName = slash + 1;
 	oldlump.resourceId = id;
-	oldlump.flags = LUMPF_FULLPATH;
+	oldlump.flags = RESFF_FULLPATH;
 	FileInfo.push_back(oldlump);
 	return true;
 }

source/common/filesystem/source/fs_stringpool.cpp

@@ -104,10 +104,11 @@ StringPool::Block *StringPool::AddBlock(size_t size)
 	return mem;
 }
 
-void *StringPool::iAlloc(size_t size)
+void *StringPool::Alloc(size_t size)
 {
 	Block *block;
 
+	size = (size + 7) & ~7;
 	for (block = TopBlock; block != nullptr; block = block->NextBlock)
 	{
 		void *res = block->Alloc(size);
@@ -122,7 +123,7 @@ void *StringPool::iAlloc(size_t size)
 
 const char* StringPool::Strdup(const char* str)
 {
-	char* p = (char*)iAlloc((strlen(str) + 8) & ~7 );
+	char* p = (char*)Alloc(strlen(str) + 1);
 	strcpy(p, str);
 	return p;
 }

source/common/filesystem/source/fs_stringpool.h

@@ -12,12 +12,12 @@ private:
 public:
 	~StringPool();
 	const char* Strdup(const char*);
+	void* Alloc(size_t size);
 
 protected:
 	struct Block;
 
 	Block *AddBlock(size_t size);
-	void *iAlloc(size_t size);
 
 	Block *TopBlock;
 	Block *FreeBlocks;

source/common/filesystem/source/resourcefile.cpp

@@ -35,13 +35,29 @@
 */
 
 #include <miniz.h>
-#include "resourcefile_internal.h"
+#include "resourcefile.h"
 #include "md5.hpp"
 #include "fs_stringpool.h"
 #include "files_internal.h"
+#include "unicode.h"
+#include "fs_findfile.h"
+#include "fs_decompress.h"
+#include "wildcards.hpp"
 
 namespace FileSys {
 	
+// this is for restricting shared file readers to the main thread.
+thread_local bool mainThread;
+void SetMainThread()
+{
+	// only set the global flag on the first thread calling this.
+	static bool done = false;
+	if (!done)
+	{
+		mainThread = done = true;
+	}
+}
+
 std::string ExtractBaseName(const char* path, bool include_extension)
 {
 	const char* src, * dot;
@@ -78,195 +94,43 @@ void strReplace(std::string& str, const char *from, const char* to)
 	}
 }
 
-//==========================================================================
-//
-// File reader that reads from a lump's cache
-//
-//==========================================================================
-
-class FLumpReader : public MemoryReader
-{
-	FResourceLump *source;
-
-public:
-	FLumpReader(FResourceLump *src)
-		: MemoryReader(NULL, src->LumpSize), source(src)
-	{
-		src->Lock();
-		bufptr = src->Cache;
-	}
-
-	~FLumpReader()
-	{
-		source->Unlock();
-	}
-};
-
-
-//==========================================================================
-//
-// Base class for resource lumps
-//
-//==========================================================================
-
-FResourceLump::~FResourceLump()
-{
-	if (Cache != NULL && RefCount >= 0)
-	{
-		delete [] Cache;
-		Cache = NULL;
-	}
-	Owner = NULL;
-}
-
-
-//==========================================================================
-//
-// Sets up the lump name information for anything not coming from a WAD file.
-//
-//==========================================================================
-
-void FResourceLump::LumpNameSetup(const char *iname, StringPool* allocator)
-{
-	// this causes interference with real Dehacked lumps.
-	if (!stricmp(iname, "dehacked.exe"))
-	{
-		iname = "";
-	}
-	else if (allocator)
-	{
-		iname = allocator->Strdup(iname);
-	}
-
-	FullName = iname;
-}
-
 //==========================================================================
 //
 // Checks for embedded resource files
 //
 //==========================================================================
 
-static bool IsWadInFolder(const FResourceFile* const archive, const char* const resPath)
+bool FResourceFile::IsFileInFolder(const char* const resPath)
 {
 	// Checks a special case when <somefile.wad> was put in
 	// <myproject> directory inside <myproject.zip>
 
-	if (NULL == archive)
+    const auto dirName = ExtractBaseName(FileName);
-	{
-		return false;
-	}
-
-    const auto dirName = ExtractBaseName(archive->GetFileName());
 	const auto fileName = ExtractBaseName(resPath, true);
 	const std::string filePath = dirName + '/' + fileName;
 
 	return 0 == stricmp(filePath.c_str(), resPath);
 }
 
-void FResourceLump::CheckEmbedded(LumpFilterInfo* lfi)
+void FResourceFile::CheckEmbedded(uint32_t entry, LumpFilterInfo* lfi)
 {
 	// Checks for embedded archives
-	const char *c = strstr(FullName, ".wad");
+	auto FullName = Entries[entry].FileName;
-	if (c && strlen(c) == 4 && (!strchr(FullName, '/') || IsWadInFolder(Owner, FullName)))
+	const char *c = strstr(FullName, ".wad"); // fixme: Use lfi for this.
+	if (c && strlen(c) == 4 && (!strchr(FullName, '/') || IsFileInFolder(FullName)))
 	{
-		Flags |= LUMPF_EMBEDDED;
+		Entries[entry].Flags |= RESFF_EMBEDDED;
 	}
 	else if (lfi) for (auto& fstr : lfi->embeddings)
 	{
 		if (!stricmp(FullName, fstr.c_str()))
 		{
-			Flags |= LUMPF_EMBEDDED;
+			Entries[entry].Flags |= RESFF_EMBEDDED;
 		}
 	}
 }
 
 
-//==========================================================================
-//
-// this is just for completeness. For non-Zips only an uncompressed lump can
-// be returned.
-//
-//==========================================================================
-
-FCompressedBuffer FResourceLump::GetRawData()
-{
-	FCompressedBuffer cbuf = { (unsigned)LumpSize, (unsigned)LumpSize, METHOD_STORED, 0, 0, new char[LumpSize] };
-	memcpy(cbuf.mBuffer, Lock(), LumpSize);
-	Unlock();
-	cbuf.mCRC32 = crc32(0, (uint8_t*)cbuf.mBuffer, LumpSize);
-	return cbuf;
-}
-
-
-//==========================================================================
-//
-// Returns the owner's FileReader if it can be used to access this lump
-//
-//==========================================================================
-
-FileReader *FResourceLump::GetReader()
-{
-	return NULL;
-}
-
-//==========================================================================
-//
-// Returns a file reader to the lump's cache
-//
-//==========================================================================
-
-FileReader FResourceLump::NewReader()
-{
-	return FileReader(new FLumpReader(this));
-}
-
-//==========================================================================
-//
-// Caches a lump's content and increases the reference counter
-//
-//==========================================================================
-
-void *FResourceLump::Lock()
-{
-	if (Cache != NULL)
-	{
-		if (RefCount > 0) RefCount++;
-	}
-	else if (LumpSize > 0)
-	{
-		try
-		{
-			FillCache();
-		}
-		catch (const FileSystemException& err)
-		{
-			// enrich the message with info about this lump.
-			throw FileSystemException("%s, file '%s': %s", getName(), Owner->GetFileName(), err.what());
-		}
-	}
-	return Cache;
-}
-
-//==========================================================================
-//
-// Decrements reference counter and frees lump if counter reaches 0
-//
-//==========================================================================
-
-int FResourceLump::Unlock()
-{
-	if (LumpSize > 0 && RefCount > 0)
-	{
-		if (--RefCount == 0)
-		{
-			delete [] Cache;
-			Cache = NULL;
-		}
-	}
-	return RefCount;
-}
-
 //==========================================================================
 //
 // Opens a resource file
@@ -282,11 +146,13 @@ FResourceFile *CheckPak(const char *filename, FileReader &file, LumpFilterInfo*
 FResourceFile *CheckZip(const char *filename, FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 FResourceFile *Check7Z(const char *filename,  FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 FResourceFile* CheckSSI(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
+FResourceFile* CheckHog(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
+FResourceFile* CheckMvl(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 FResourceFile* CheckWHRes(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 FResourceFile *CheckLump(const char *filename,FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 FResourceFile *CheckDir(const char *filename, bool nosub, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 
-static CheckFunc funcs[] = { CheckWad, CheckZip, Check7Z, CheckPak, CheckGRP, CheckRFF, CheckSSI, CheckWHRes, CheckLump };
+static CheckFunc funcs[] = { CheckWad, CheckZip, Check7Z, CheckPak, CheckGRP, CheckRFF, CheckSSI, CheckHog, CheckMvl, CheckWHRes, CheckLump };
 
 static int nulPrintf(FSMessageLevel msg, const char* fmt, ...)
 {
@@ -347,11 +213,95 @@ FResourceFile::~FResourceFile()
 	if (!stringpool->shared) delete stringpool;
 }
 
-int lumpcmp(const void * a, const void * b)
+//==========================================================================
+//
+// this is just for completeness. For non-Zips only an uncompressed lump can
+// be returned.
+//
+//==========================================================================
+
+FCompressedBuffer FResourceFile::GetRawData(uint32_t entry)
 {
-	FResourceLump * rec1 = (FResourceLump *)a;
+	size_t LumpSize = entry << NumLumps ? Entries[entry].Length : 0;
-	FResourceLump * rec2 = (FResourceLump *)b;
+	FCompressedBuffer cbuf = { LumpSize, LumpSize, METHOD_STORED, 0, 0, LumpSize == 0? nullptr : new char[LumpSize] };
-	return stricmp(rec1->getName(), rec2->getName());
+	if (LumpSize > 0)
+	{
+		auto fr = GetEntryReader(entry, READER_SHARED, 0);
+		size_t read = fr.Read(cbuf.mBuffer, LumpSize);
+		if (read < LumpSize)
+		{
+			delete cbuf.mBuffer;
+			cbuf.mBuffer = nullptr;
+			LumpSize = cbuf.mCompressedSize = cbuf.mSize = 0;
+		}
+	}
+	if (LumpSize > 0)
+		cbuf.mCRC32 = crc32(0, (uint8_t*)cbuf.mBuffer, LumpSize);
+	return cbuf;
+}
+
+
+//==========================================================================
+//
+// normalize the visible file name in the system
+// to lowercase canonical precomposed Unicode.
+//
+//==========================================================================
+
+const char* FResourceFile::NormalizeFileName(const char* fn, int fallbackcp)
+{
+	if (!fn || !*fn) return "";
+	auto norm = tolower_normalize(fn);
+	if (!norm)
+	{
+		if (fallbackcp == 437)
+		{
+			std::vector<char> buffer;
+			ibm437_to_utf8(fn, buffer);
+			norm = tolower_normalize(buffer.data());
+		}
+		// maybe handle other codepages
+		else
+		{
+			// if the filename is not valid UTF-8, nuke all bytes larger than 0x80 so that we still got something semi-usable
+			std::string ffn = fn;
+			for (auto& c : ffn)
+			{
+				if (c & 0x80) c = '@';
+			}
+			norm = tolower_normalize(&ffn.front());
+		}
+	}
+	FixPathSeparator(norm);
+	auto pooled = stringpool->Strdup(norm);
+	free(norm);
+	return pooled;
+
+}
+
+//==========================================================================
+//
+// allocate the Entries array
+// this also uses the string pool to reduce maintenance
+//
+//==========================================================================
+
+FResourceEntry* FResourceFile::AllocateEntries(int count)
+{
+	NumLumps = count;
+	Entries = (FResourceEntry*)stringpool->Alloc(count * sizeof(FResourceEntry));
+	memset(Entries, 0, count * sizeof(FResourceEntry));
+	return Entries;
+}
+
+
+//---------------------------------------------------
+int entrycmp(const void* a, const void* b)
+{
+	FResourceEntry* rec1 = (FResourceEntry*)a;
+	FResourceEntry* rec2 = (FResourceEntry*)b;
+	// we are comparing lowercase UTF-8 here 
+	return strcmp(rec1->FileName, rec2->FileName);
 }
 
 //==========================================================================
@@ -400,30 +350,132 @@ void FResourceFile::GenerateHash()
 //
 //==========================================================================
 
-void FResourceFile::PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterInfo *filter)
+void FResourceFile::PostProcessArchive(LumpFilterInfo *filter)
 {
-	// Entries in archives are sorted alphabetically
+	// only do this for archive types which contain full file names. All others are assumed to be pre-sorted.
-	qsort(lumps, NumLumps, lumpsize, lumpcmp);
+	if (NumLumps == 0 || !(Entries[0].Flags & RESFF_FULLPATH)) return;
+
+	// First eliminate all unwanted files
+	if (filter)
+	{
+		for (uint32_t i = 0; i < NumLumps; i++)
+		{
+			std::string name = Entries[i].FileName;
+			for (auto& pattern : filter->blockednames)
+			{
+				if (wildcards::match(name, pattern))
+				{
+					Entries[i].FileName = "";
+					continue;
+				}
+			}
+		}
+	}
+
+	// Entries in archives are sorted alphabetically.
+	qsort(Entries, NumLumps, sizeof(Entries[0]), entrycmp);
 	if (!filter) return;
+	FindCommonFolder(filter);
 
 	// Filter out lumps using the same names as the Autoload.* sections
-	// in the ini file use. We reduce the maximum lump concidered after
+	// in the ini file. We reduce the maximum lump concidered after
 	// each one so that we don't risk refiltering already filtered lumps.
 	uint32_t max = NumLumps;
-	max -= FilterLumpsByGameType(filter, lumps, lumpsize, max);
 
+	for (auto& LumpFilter : filter->gameTypeFilter)
+	{
 		ptrdiff_t len;
 		ptrdiff_t lastpos = -1;
 		std::string file;
-	std::string& LumpFilter = filter->dotFilter;
+		while (size_t(len = LumpFilter.find_first_of('.', lastpos + 1)) != LumpFilter.npos)
-	while (size_t(len = LumpFilter.find_first_of('.', lastpos+1)) != LumpFilter.npos)
 		{
-		max -= FilterLumps(std::string(LumpFilter, 0, len), lumps, lumpsize, max);
+			max -= FilterLumps(std::string(LumpFilter, 0, len), max);
 			lastpos = len;
 		}
-	max -= FilterLumps(LumpFilter, lumps, lumpsize, max);
+		max -= FilterLumps(LumpFilter, max);
+	}
 
-	JunkLeftoverFilters(lumps, lumpsize, max);
+	JunkLeftoverFilters(max);
+
+	for (uint32_t i = 0; i < NumLumps; i++)
+	{
+		CheckEmbedded(i, filter);
+	}
+
+}
+
+//==========================================================================
+//
+// FResourceFile :: FindCommonFolder
+//
+// Checks if all content is in a common folder that can be stripped out.
+//
+//==========================================================================
+
+void FResourceFile::FindCommonFolder(LumpFilterInfo* filter)
+{
+	std::string name0, name1;
+	bool foundspeciallump = false;
+	bool foundprefix = false;
+
+	// try to find a path prefix.
+	for (uint32_t i = 0; i < NumLumps; i++)
+	{
+		if (*Entries[i].FileName == 0) continue;
+		std::string name = Entries[i].FileName;
+
+		// first eliminate files we do not want to have.
+		// Some, like MacOS resource forks and executables are eliminated unconditionally, but the calling code can alsp pass a list of invalid content.
+		if (name.find("filter/") == 0)
+			return; // 'filter' is a reserved name of the file system. If this appears in the root we got no common folder, and 'filter' cannot be it.
+
+		if (!foundprefix)
+		{
+			// check for special names, if one of these gets found this must be treated as a normal zip.
+			bool isspecial = name.find("/") == std::string::npos ||
+				(filter && std::find(filter->reservedFolders.begin(), filter->reservedFolders.end(), name) != filter->reservedFolders.end());
+			if (isspecial) break;
+			name0 = std::string(name, 0, name.rfind("/") + 1);
+			name1 = std::string(name, 0, name.find("/") + 1);
+			foundprefix = true;
+		}
+
+		if (name.find(name0) != 0)
+		{
+			if (!name1.empty())
+			{
+				name0 = name1;
+				if (name.find(name0) != 0)
+				{
+					name0 = "";
+				}
+			}
+			if (name0.empty())
+				break;
+		}
+		if (!foundspeciallump && filter)
+		{
+			// at least one of the more common definition lumps must be present.
+			for (auto& p : filter->requiredPrefixes)
+			{
+				if (name.find(name0 + p) == 0 || name.rfind(p) == size_t(name.length() - p.length()))
+				{
+					foundspeciallump = true;
+					break;
+				}
+			}
+		}
+	}
+	// If it ran through the list without finding anything it should not attempt any path remapping.
+	if (!foundspeciallump || name0.empty()) return;
+
+	size_t pathlen = name0.length();
+	for (uint32_t i = 0; i < NumLumps; i++)
+	{
+		if (Entries[i].FileName[0] == 0) continue;
+		Entries[i].FileName += pathlen;
+
+	}
 }
 
 //==========================================================================
@@ -436,7 +488,7 @@ void FResourceFile::PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterI
 //
 //==========================================================================
 
-int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_t lumpsize, uint32_t max)
+int FResourceFile::FilterLumps(const std::string& filtername, uint32_t max)
 {
 	uint32_t start, end;
 
@@ -446,44 +498,35 @@ int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_
 	}
 	std::string filter = "filter/" + filtername + '/';
 
-	bool found = FindPrefixRange(filter.c_str(), lumps, lumpsize, max, start, end);
+	bool found = FindPrefixRange(filter.c_str(), max, start, end);
-
-	// Workaround for old Doom filter names (todo: move out of here.)
-	if (!found && filtername.find("doom.id.doom") == 0)
-	{
-		strReplace(filter, "doom.id.doom", "doom.doom");
-		found = FindPrefixRange(filter.c_str(), lumps, lumpsize, max, start, end);
-	}
 
 	if (found)
 	{
-		void *from = (uint8_t *)lumps + start * lumpsize;
 
 		// Remove filter prefix from every name
-		void *lump_p = from;
+		for (uint32_t i = start; i < end; ++i)
-		for (uint32_t i = start; i < end; ++i, lump_p = (uint8_t *)lump_p + lumpsize)
 		{
-			FResourceLump *lump = (FResourceLump *)lump_p;
+			assert(strnicmp(Entries[i].FileName, filter.c_str(), filter.length()) == 0);
-			assert(strnicmp(lump->FullName, filter.c_str(), filter.length()) == 0);
+			Entries[i].FileName += filter.length();
-			lump->LumpNameSetup(lump->FullName + filter.length(), nullptr);
 		}
 
 		// Move filtered lumps to the end of the lump list.
-		size_t count = (end - start) * lumpsize;
+		size_t count = (end - start);
-		void *to = (uint8_t *)lumps + NumLumps * lumpsize - count;
+		auto from = Entries + start;
+		auto to = Entries + NumLumps - count;
 		assert (to >= from);
 
 		if (from != to)
 		{
 			// Copy filtered lumps to a temporary buffer.
-			uint8_t *filteredlumps = new uint8_t[count];
+			auto filteredlumps = new FResourceEntry[count];
-			memcpy(filteredlumps, from, count);
+			memcpy(filteredlumps, from, count * sizeof(*Entries));
 
 			// Shift lumps left to make room for the filtered ones at the end.
-			memmove(from, (uint8_t *)from + count, (NumLumps - end) * lumpsize);
+			memmove(from, from + count, (NumLumps - end) * sizeof(*Entries));
 
 			// Copy temporary buffer to newly freed space.
-			memcpy(to, filteredlumps, count);
+			memcpy(to, filteredlumps, count * sizeof(*Entries));
 
 			delete[] filteredlumps;
 		}
@@ -491,29 +534,6 @@ int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_
 	return end - start;
 }
 
-//==========================================================================
-//
-// FResourceFile :: FilterLumpsByGameType
-//
-// Matches any lumps that match "filter/game-<gametype>/*". Includes
-// inclusive gametypes like Raven.
-//
-//==========================================================================
-
-int FResourceFile::FilterLumpsByGameType(LumpFilterInfo *filter, void *lumps, size_t lumpsize, uint32_t max)
-{
-	if (filter == nullptr)
-	{
-		return 0;
-	}
-	int count = 0;
-	for (auto &fstring : filter->gameTypeFilter)
-	{
-		count += FilterLumps(fstring, lumps, lumpsize, max);
-	}
-	return count;
-}
-
 //==========================================================================
 //
 // FResourceFile :: JunkLeftoverFilters
@@ -522,19 +542,17 @@ int FResourceFile::FilterLumpsByGameType(LumpFilterInfo *filter, void *lumps, si
 //
 //==========================================================================
 
-void FResourceFile::JunkLeftoverFilters(void *lumps, size_t lumpsize, uint32_t max)
+void FResourceFile::JunkLeftoverFilters(uint32_t max)
 {
 	uint32_t start, end;
-	if (FindPrefixRange("filter/", lumps, lumpsize, max, start, end))
+	if (FindPrefixRange("filter/", max, start, end))
 	{
 		// Since the resource lumps may contain non-POD data besides the
 		// full name, we "delete" them by erasing their names so they
 		// can't be found.
-		void *stop = (uint8_t *)lumps + end * lumpsize;
+		for (uint32_t i = start; i < end; i++)
-		for (void *p = (uint8_t *)lumps + start * lumpsize; p < stop; p = (uint8_t *)p + lumpsize)
 		{
-			FResourceLump *lump = (FResourceLump *)p;
+			Entries[i].FileName = "";
-			lump->clearName();
 		}
 	}
 }
@@ -549,25 +567,24 @@ void FResourceFile::JunkLeftoverFilters(void *lumps, size_t lumpsize, uint32_t m
 //
 //==========================================================================
 
-bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lumpsize, uint32_t maxlump, uint32_t &start, uint32_t &end)
+bool FResourceFile::FindPrefixRange(const char* filter, uint32_t maxlump, uint32_t &start, uint32_t &end)
 {
 	uint32_t min, max, mid, inside;
-	FResourceLump *lump;
 	int cmp = 0;
 
 	end = start = 0;
 
 	// Pretend that our range starts at 1 instead of 0 so that we can avoid
 	// unsigned overflow if the range starts at the first lump.
-	lumps = (uint8_t *)lumps - lumpsize;
+	auto lumps = &Entries[-1];
 
 	// Binary search to find any match at all.
 	mid = min = 1, max = maxlump;
 	while (min <= max)
 	{
 		mid = min + (max - min) / 2;
-		lump = (FResourceLump *)((uint8_t *)lumps + mid * lumpsize);
+		auto lump = &lumps[mid];
-		cmp = strnicmp(lump->FullName, filter, (int)strlen(filter));
+		cmp = strnicmp(lump->FileName, filter, strlen(filter));
 		if (cmp == 0)
 			break;
 		else if (cmp < 0)
@@ -586,8 +603,8 @@ bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lump
 	while (min <= max)
 	{
 		mid = min + (max - min) / 2;
-		lump = (FResourceLump *)((uint8_t *)lumps + mid * lumpsize);
+		auto lump = &lumps[mid];
-		cmp = strnicmp(lump->FullName, filter, (int)strlen(filter));
+		cmp = strnicmp(lump->FileName, filter, strlen(filter));
 		// Go left on matches and right on misses.
 		if (cmp == 0)
 			max = mid - 1;
@@ -601,8 +618,8 @@ bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lump
 	while (min <= max)
 	{
 		mid = min + (max - min) / 2;
-		lump = (FResourceLump *)((uint8_t *)lumps + mid * lumpsize);
+		auto lump = &lumps[mid];
-		cmp = strnicmp(lump->FullName, filter, (int)strlen(filter));
+		cmp = strnicmp(lump->FileName, filter, strlen(filter));
 		// Go right on matches and left on misses.
 		if (cmp == 0)
 			min = mid + 1;
@@ -621,27 +638,19 @@ bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lump
 
 int FResourceFile::FindEntry(const char *name)
 {
+	auto norm_fn = tolower_normalize(name);
 	for (unsigned i = 0; i < NumLumps; i++)
 	{
-		if (!stricmp(name, getName(i)))
+		if (!strcmp(norm_fn, getName(i)))
 		{
+			free(norm_fn);
 			return i;
 		}
 	}
+	free(norm_fn);
 	return -1;
 }
 
-//==========================================================================
-//
-// Caches a lump's content and increases the reference counter
-//
-//==========================================================================
-
-FileReader *FUncompressedLump::GetReader()
-{
-	Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
-	return Owner->GetContainerReader();
-}
 
 //==========================================================================
 //
@@ -649,103 +658,73 @@ FileReader *FUncompressedLump::GetReader()
 //
 //==========================================================================
 
-int FUncompressedLump::FillCache()
+FileReader FResourceFile::GetEntryReader(uint32_t entry, int readertype, int readerflags)
 {
-	const char * buffer = Owner->GetContainerReader()->GetBuffer();
+	FileReader fr;
-
+	if (entry < NumLumps)
-	if (buffer != NULL)
 	{
-		// This is an in-memory file so the cache can point directly to the file's data.
+		if (Entries[entry].Flags & RESFF_NEEDFILESTART)
-		Cache = const_cast<char*>(buffer) + Position;
+		{
-		RefCount = -1;
+			SetEntryAddress(entry);
-		return -1;
 		}
-
+		if (!(Entries[entry].Flags & RESFF_COMPRESSED))
-	Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
-	Cache = new char[LumpSize];
-
-	auto read = Owner->GetContainerReader()->Read(Cache, LumpSize);
-	if (read != LumpSize)
 		{
-		throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize);
+			auto buf = Reader.GetBuffer();
-	}
+			// if this is backed by a memory buffer, create a new reader directly referencing it.
-
+			if (buf != nullptr)
-	RefCount = 1;
-	return 1;
-}
-
-//==========================================================================
-//
-// Base class for uncompressed resource files
-//
-//==========================================================================
-
-FUncompressedFile::FUncompressedFile(const char *filename, StringPool* sp)
-: FResourceFile(filename, sp)
-{}
-
-FUncompressedFile::FUncompressedFile(const char *filename, FileReader &r, StringPool* sp)
-	: FResourceFile(filename, r, sp)
-{}
-
-
-//==========================================================================
-//
-// external lump
-//
-//==========================================================================
-
-FExternalLump::FExternalLump(const char *_filename, int filesize, StringPool* stringpool)
-{
-	FileName = stringpool->Strdup(_filename);
-
-	if (filesize == -1)
 			{
-		FileReader f;
+				fr.OpenMemory(buf + Entries[entry].Position, Entries[entry].Length);
-
-		if (f.OpenFile(_filename))
-		{
-			LumpSize = (int)f.GetLength();
 			}
 			else
 			{
-			LumpSize = 0;
+				if (readertype == READER_SHARED && !mainThread)
+					readertype = READER_NEW;
+				if (readertype == READER_SHARED)
+				{
+					fr.OpenFilePart(Reader, Entries[entry].Position, Entries[entry].Length);
+				}
+				else if (readertype == READER_NEW)
+				{
+					fr.OpenFile(FileName, Entries[entry].Position, Entries[entry].Length);
+				}
+				else if (readertype == READER_CACHED)
+				{
+					Reader.Seek(Entries[entry].Position, FileReader::SeekSet);
+					auto data = Reader.Read(Entries[entry].Length);
+					fr.OpenMemoryArray(data);
+				}
 			}
 		}
 		else
 		{
-		LumpSize = filesize;
+			FileReader fri;
+			if (readertype == READER_NEW || !mainThread) fri.OpenFile(FileName, Entries[entry].Position, Entries[entry].CompressedSize);
+			else fri.OpenFilePart(Reader, Entries[entry].Position, Entries[entry].CompressedSize);
+			int flags = DCF_TRANSFEROWNER | DCF_EXCEPTIONS;
+			if (readertype == READER_CACHED) flags |= DCF_CACHED;
+			else if (readerflags & READERFLAG_SEEKABLE) flags |= DCF_SEEKABLE;
+			OpenDecompressor(fr, fri, Entries[entry].Length, Entries[entry].Method, flags);
 		}
+	}
+	return fr;
 }
 
-
+FileData FResourceFile::Read(int entry)
-//==========================================================================
-//
-// Caches a lump's content and increases the reference counter
-// For external lumps this reopens the file each time it is accessed
-//
-//==========================================================================
-
-int FExternalLump::FillCache()
 {
-	Cache = new char[LumpSize];
+	if (!(Entries[entry].Flags & RESFF_COMPRESSED))
-	FileReader f;
+	{
+		auto buf = Reader.GetBuffer();
+		// if this is backed by a memory buffer, we can just return a reference to the backing store.
+		if (buf != nullptr)
+		{
+			return FileData(buf + Entries[entry].Position, Entries[entry].Length, false);
+		}
+	}
 
-	if (f.OpenFile(FileName))
+	auto fr = GetEntryReader(entry, READER_SHARED, 0);
-	{
+	return fr.Read(entry < NumLumps ? Entries[entry].Length : 0);
-		auto read = f.Read(Cache, LumpSize);
-		if (read != LumpSize)
-		{
-			throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize);
-		}
-	}
-	else
-	{
-		throw FileSystemException("unable to open file");
-	}
-	RefCount = 1;
-	return 1;
 }
 
 
+
 }

source/common/filesystem/source/resourcefile_internal.h

@@ -1,38 +0,0 @@
-#pragma once
-
-#include "resourcefile.h"
-
-namespace FileSys {
-struct FUncompressedLump : public FResourceLump
-{
-	int				Position;
-
-	virtual FileReader *GetReader();
-	virtual int FillCache() override;
-	virtual int GetFileOffset() { return Position; }
-
-};
-
-// Base class for uncompressed resource files (WAD, GRP, PAK and single lumps)
-class FUncompressedFile : public FResourceFile
-{
-protected:
-	TArray<FUncompressedLump> Lumps;
-
-	FUncompressedFile(const char *filename, StringPool* sp);
-	FUncompressedFile(const char *filename, FileReader &r, StringPool* sp);
-	virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; }
-};
-
-
-// should only be used internally.
-struct FExternalLump : public FResourceLump
-{
-	const char* FileName;
-
-	FExternalLump(const char *_filename, int filesize, StringPool* sp);
-	virtual int FillCache() override;
-
-};
-
-}

source/common/filesystem/source/unicode.cpp

@@ -121,7 +121,7 @@ void ibm437_to_utf8(const char* in, std::vector<char>& buffer)
 
 	while (int char1 = (uint8_t)*in++)
 	{
-		if (char1 >= 0x80) char1 = ibm437map[char1];
+		if (char1 >= 0x80) char1 = ibm437map[char1 - 0x80];
 		utf8_encode(char1, buffer);
 	}
 	buffer.push_back(0);
@@ -140,4 +140,22 @@ char *tolower_normalize(const char *str)
 	return (char*)retval;
 }
 
+//==========================================================================
+//
+// validates the string for proper UTF-8
+//
+//==========================================================================
+
+bool unicode_validate(const char* str)
+{
+	while (*str != 0)
+	{
+		int cp;
+		auto result = utf8proc_iterate((const uint8_t*)str, -1, &cp);
+		if (result < 0) return false;
+	}
+	return true;
+}
+
+
 }

source/common/filesystem/source/unicode.h

@@ -6,7 +6,7 @@ namespace FileSys {
 	
 void utf16_to_utf8(const unsigned short* in, std::vector<char>& buffer);
 void ibm437_to_utf8(const char* in, std::vector<char>& buffer);
-int unicode_tolower(int c);
 char *tolower_normalize(const char *str);
+bool unicode_validate(const char* str);
 
 }

source/common/fonts/singlelumpfont.cpp

@@ -125,7 +125,7 @@ FSingleLumpFont::FSingleLumpFont (const char *name, int lump) : FFont(lump)
 	FontName = name;
 
 	auto data1 = fileSystem.ReadFile (lump);
-	auto data = data1.GetBytes();
+	auto data = data1.bytes();
 
 	if (data[0] == 0xE1 && data[1] == 0xE6 && data[2] == 0xD5 && data[3] == 0x1A)
 	{
@@ -475,7 +475,7 @@ void FSingleLumpFont::LoadBMF(int lump, const uint8_t *data)
 void FSingleLumpFont::CheckFON1Chars()
 {
 	auto memLump = fileSystem.ReadFile(Lump);
-	auto data = memLump.GetBytes();
+	auto data = memLump.bytes();
 	const uint8_t* data_p;
 
 	data_p = data + 8;

source/common/menu/savegamemanager.cpp

@@ -316,7 +316,7 @@ unsigned FSavegameManagerBase::ExtractSaveData(int index)
 		auto pic = resf->FindEntry("savepic.png");
 		if (pic >= 0)
 		{
-			FileReader picreader = resf->GetEntryReader(pic);
+			FileReader picreader = resf->GetEntryReader(pic, FileSys::READER_NEW, FileSys::READERFLAG_SEEKABLE);
 			PNGHandle *png = M_VerifyPNG(picreader);
 			if (png != nullptr)
 			{

source/common/models/model.cpp

@@ -165,7 +165,7 @@ unsigned FindModel(const char * path, const char * modelfile, bool silent)
 
 	int len = fileSystem.FileLength(lump);
 	auto lumpd = fileSystem.ReadFile(lump);
-	const char * buffer = lumpd.GetString();
+	const char * buffer = lumpd.string();
 
 	if ( (size_t)fullname.LastIndexOf("_d.3d") == fullname.Len()-5 )
 	{

source/common/models/models_iqm.cpp

@@ -274,7 +274,7 @@ void IQMModel::LoadGeometry()
 	try
 	{
 		auto lumpdata = fileSystem.ReadFile(mLumpNum);
-		IQMFileReader reader(lumpdata.GetMem(), (int)lumpdata.GetSize());
+		IQMFileReader reader(lumpdata.data(), (int)lumpdata.size());
 
 		Vertices.Resize(NumVertices);
 		for (IQMVertexArray& vertexArray : VertexArrays)

source/common/models/models_md2.cpp

@@ -178,7 +178,7 @@ void FDMDModel::LoadGeometry()
 {
 	static int axis[3] = { VX, VY, VZ };
 	auto lumpdata = fileSystem.ReadFile(mLumpNum);
-	auto buffer = lumpdata.GetString();
+	auto buffer = lumpdata.string();
 	texCoords = new FTexCoord[info.numTexCoords];
 	memcpy(texCoords, buffer + info.offsetTexCoords, info.numTexCoords * sizeof(FTexCoord));
 
@@ -502,7 +502,7 @@ void FMD2Model::LoadGeometry()
 	static int axis[3] = { VX, VY, VZ };
 	uint8_t   *md2_frames;
 	auto lumpdata = fileSystem.ReadFile(mLumpNum);
-	auto buffer = lumpdata.GetString();
+	auto buffer = lumpdata.string();
 
 	texCoords = new FTexCoord[info.numTexCoords];
 	memcpy(texCoords, (uint8_t*)buffer + info.offsetTexCoords, info.numTexCoords * sizeof(FTexCoord));

source/common/models/models_md3.cpp

@@ -190,7 +190,7 @@ bool FMD3Model::Load(const char * path, int lumpnum, const char * buffer, int le
 void FMD3Model::LoadGeometry()
 {
 	auto lumpdata = fileSystem.ReadFile(mLumpNum);
-	auto buffer = lumpdata.GetString();
+	auto buffer = lumpdata.string();
 	md3_header_t * hdr = (md3_header_t *)buffer;
 	md3_surface_t * surf = (md3_surface_t*)(buffer + LittleLong(hdr->Ofs_Surfaces));
 

source/common/models/models_ue1.cpp

@@ -71,9 +71,9 @@ void FUE1Model::LoadGeometry()
 {
 	const char *buffer, *buffer2;
 	auto lump =  fileSystem.ReadFile(mDataLump);
-	buffer = lump.GetString();
+	buffer = lump.string();
 	auto lump2 =  fileSystem.ReadFile(mAnivLump);
-	buffer2 = lump2.GetString();
+	buffer2 = lump2.string();
 	// map structures
 	dhead = (const d3dhead*)(buffer);
 	dpolys = (const d3dpoly*)(buffer+sizeof(d3dhead));

source/common/models/voxels.cpp

@@ -162,8 +162,8 @@ FVoxel *R_LoadKVX(int lumpnum)
 	int i, j, n;
 
 	auto lump =  fileSystem.ReadFile(lumpnum);	// FileData adds an extra 0 byte to the end.
-	auto rawvoxel = lump.GetBytes();
+	auto rawvoxel = lump.bytes();
-	int voxelsize = (int)(lump.GetSize());
+	int voxelsize = (int)(lump.size());
 
 	// Oh, KVX, why couldn't you have a proper header? We'll just go through
 	// and collect each MIP level, doing lots of range checking, and if the

source/common/rendering/gl/gl_shaderprogram.cpp

@@ -89,7 +89,6 @@ void FShaderProgram::Compile(ShaderType type, const char *lumpName, const char *
 {
 	int lump = fileSystem.CheckNumForFullName(lumpName);
 	if (lump == -1) I_FatalError("Unable to load '%s'", lumpName);
-	auto sp = fileSystem.ReadFile(lump);
 	FString code = GetStringFromLump(lump);
 
 	Compile(type, lumpName, code, defines, maxGlslVersion);

source/common/rendering/gles/gles_shader.cpp

@@ -383,11 +383,9 @@ bool FShader::Load(const char * name, const char * vert_prog_lump_, const char *
 
 	int vp_lump = fileSystem.CheckNumForFullName(vert_prog_lump.GetChars(), 0);
 	if (vp_lump == -1) I_Error("Unable to load '%s'", vert_prog_lump.GetChars());
-	auto vp_data = fileSystem.ReadFile(vp_lump);
 
 	int fp_lump = fileSystem.CheckNumForFullName(frag_prog_lump.GetChars(), 0);
 	if (fp_lump == -1) I_Error("Unable to load '%s'", frag_prog_lump.GetChars());
-	auto fp_data = fileSystem.ReadFile(fp_lump);
 
 
 
@@ -422,7 +420,6 @@ bool FShader::Load(const char * name, const char * vert_prog_lump_, const char *
 		{
 			int pp_lump = fileSystem.CheckNumForFullName(proc_prog_lump.GetChars());
 			if (pp_lump == -1) I_Error("Unable to load '%s'", proc_prog_lump.GetChars());
-			auto ppf = fileSystem.ReadFile(pp_lump);
 			FString pp_data = GetStringFromLump(pp_lump);
 
 			if (pp_data.IndexOf("ProcessMaterial") < 0 && pp_data.IndexOf("SetupMaterial") < 0)

source/common/rendering/gles/gles_shaderprogram.cpp

@@ -89,7 +89,6 @@ void FShaderProgram::Compile(ShaderType type, const char *lumpName, const char *
 {
 	int lump = fileSystem.CheckNumForFullName(lumpName);
 	if (lump == -1) I_FatalError("Unable to load '%s'", lumpName);
-	auto sp = fileSystem.ReadFile(lump);
 	FString code = GetStringFromLump(lump);
 	Compile(type, lumpName, code, defines, maxGlslVersion);
 }

source/common/rendering/vulkan/shaders/vk_ppshader.cpp

@@ -69,7 +69,6 @@ FString VkPPShader::LoadShaderCode(const FString &lumpName, const FString &defin
 {
 	int lump = fileSystem.CheckNumForFullName(lumpName.GetChars());
 	if (lump == -1) I_FatalError("Unable to load '%s'", lumpName.GetChars());
-	auto sp = fileSystem.ReadFile(lump);
 	FString code = GetStringFromLump(lump);
 
 	FString patchedCode;

source/common/rendering/vulkan/shaders/vk_shader.cpp

@@ -475,7 +475,6 @@ FString VkShaderManager::LoadPrivateShaderLump(const char *lumpname)
 {
 	int lump = fileSystem.CheckNumForFullName(lumpname, 0);
 	if (lump == -1) I_Error("Unable to load '%s'", lumpname);
-	auto data = fileSystem.ReadFile(lump);
 	return GetStringFromLump(lump);
 }
 

source/common/textures/formats/anmtexture.cpp

@@ -106,10 +106,10 @@ FAnmTexture::FAnmTexture (int lumpnum, int w, int h)
 void FAnmTexture::ReadFrame(uint8_t *pixels, uint8_t *palette)
 {
 	auto lump = fileSystem.ReadFile (SourceLump);
-	auto source = lump.GetBytes(); 
+	auto source = lump.bytes(); 
 
 	std::unique_ptr<anim_t> anim = std::make_unique<anim_t>(); // note that this struct is very large and should not go onto the stack!
-	if (ANIM_LoadAnim(anim.get(), source, (int)lump.GetSize()) >= 0)
+	if (ANIM_LoadAnim(anim.get(), source, (int)lump.size()) >= 0)
 	{
 		int numframes = ANIM_NumFrames(anim.get());
 		if (numframes >= 1)

source/common/textures/formats/automaptexture.cpp

@@ -93,7 +93,7 @@ PalettedPixels FAutomapTexture::CreatePalettedPixels(int conversion, int frame)
 {
 	int x, y;
 	auto data = fileSystem.ReadFile (SourceLump);
-	auto indata = data.GetBytes();
+	auto indata = data.bytes();
 
 	PalettedPixels Pixels(Width * Height);
 

source/common/textures/formats/imgztexture.cpp

@@ -121,7 +121,7 @@ FIMGZTexture::FIMGZTexture (int lumpnum, uint16_t w, uint16_t h, int16_t l, int1
 PalettedPixels FIMGZTexture::CreatePalettedPixels(int conversion, int frame)
 {
 	auto lump =  fileSystem.ReadFile (SourceLump);
-	auto imgz = (const ImageHeader *)lump.GetMem();
+	auto imgz = (const ImageHeader *)lump.data();
 	const uint8_t *data = (const uint8_t *)&imgz[1];
 
 	uint8_t *dest_p;

source/common/textures/formats/patchtexture.cpp

@@ -186,7 +186,7 @@ PalettedPixels FPatchTexture::CreatePalettedPixels(int conversion, int frame)
 	int x;
 
 	auto lump =  fileSystem.ReadFile (SourceLump);
-	const patch_t *patch = (const patch_t *)lump.GetMem();
+	const patch_t *patch = (const patch_t *)lump.data();
 
 	maxcol = (const column_t *)((const uint8_t *)patch + fileSystem.FileLength (SourceLump) - 3);
 
@@ -296,7 +296,7 @@ void FPatchTexture::DetectBadPatches ()
 	// It must be 256 pixels tall, and all its columns must have exactly
 	// one post, where each post has a supposed length of 0.
 	auto lump =  fileSystem.ReadFile (SourceLump);
-	const patch_t *realpatch = (patch_t *)lump.GetMem();
+	const patch_t *realpatch = (patch_t *)lump.data();
 	const uint32_t *cofs = realpatch->columnofs;
 	int x, x2 = LittleShort(realpatch->width);
 

source/common/textures/formats/qoitexture.cpp

@@ -141,14 +141,14 @@ int FQOITexture::CopyPixels(FBitmap *bmp, int conversion, int frame)
 	constexpr auto QOI_COLOR_HASH = [](PalEntry C) { return (C.r * 3 + C.g * 5 + C.b * 7 + C.a * 11); };
 
 	auto lump = fileSystem.ReadFile(SourceLump);
-	if (lump.GetSize() < 22) return 0;	// error
+	if (lump.size() < 22) return 0;	// error
 	PalEntry index[64] = {};
 	PalEntry pe = 0xff000000;
 
 	size_t p = 14, run = 0;
 
-	size_t chunks_len = lump.GetSize() - 8;
+	size_t chunks_len = lump.size() - 8;
-	auto bytes = lump.GetBytes();
+	auto bytes = lump.bytes();
 
 	for (int h = 0; h < Height; h++)
 	{

source/common/textures/formats/rawpagetexture.cpp

@@ -183,7 +183,7 @@ FRawPageTexture::FRawPageTexture (int lumpnum)
 PalettedPixels FRawPageTexture::CreatePalettedPixels(int conversion, int frame)
 {
 	auto lump =  fileSystem.ReadFile (SourceLump);
-	auto source = lump.GetBytes();
+	auto source = lump.bytes();
 	const uint8_t *source_p = source;
 	uint8_t *dest_p;
 
@@ -216,8 +216,8 @@ int FRawPageTexture::CopyPixels(FBitmap *bmp, int conversion, int frame)
 	{
 		auto lump =  fileSystem.ReadFile(SourceLump);
 		auto plump = fileSystem.ReadFile(mPaletteLump);
-		auto source = lump.GetBytes();
+		auto source = lump.bytes();
-		auto psource = plump.GetBytes();
+		auto psource = plump.bytes();
 		PalEntry paldata[256];
 		for (auto & pe : paldata)
 		{

source/common/textures/formats/startuptexture.cpp

@@ -165,7 +165,7 @@ FStartupTexture::FStartupTexture (int lumpnum)
 	bUseGamePalette = false;
 	
 	auto lump =  fileSystem.ReadFile (SourceLump);
-	auto source = lump.GetBytes();
+	auto source = lump.bytes();
 
 	// Initialize the bitmap palette.
 	// the palette is static so that the notches can share it.
@@ -234,7 +234,7 @@ void PlanarToChunky(T* dest, const uint8_t* src, const T* remap, int width, int
 PalettedPixels FStartupTexture::CreatePalettedPixels(int conversion, int frame)
 {
 	auto lump =  fileSystem.ReadFile (SourceLump);
-	auto source = lump.GetBytes();
+	auto source = lump.bytes();
 	const uint8_t *remap = ImageHelpers::GetRemap(conversion == luminance);
 
 
@@ -254,7 +254,7 @@ PalettedPixels FStartupTexture::CreatePalettedPixels(int conversion, int frame)
 int FStartupTexture::CopyPixels(FBitmap *bmp, int conversion, int frame)
 {
 	auto lump =  fileSystem.ReadFile (SourceLump);
-	auto source = lump.GetBytes();
+	auto source = lump.bytes();
 	PlanarToChunky((uint32_t*)bmp->GetPixels(), source + 48, startuppalette32, Width, Height);
 	return 0;
 }
@@ -282,7 +282,7 @@ FNotchTexture::FNotchTexture (int lumpnum, int width, int height)
 PalettedPixels FNotchTexture::CreatePalettedPixels(int conversion, int frame)
 {
 	auto lump =  fileSystem.ReadFile (SourceLump);
-	auto source = lump.GetBytes();
+	auto source = lump.bytes();
 	const uint8_t *remap = ImageHelpers::GetRemap(conversion == luminance);
 
 	TArray<uint8_t> Work(Width*Height, true);
@@ -305,7 +305,7 @@ PalettedPixels FNotchTexture::CreatePalettedPixels(int conversion, int frame)
 int FNotchTexture::CopyPixels(FBitmap *bmp, int conversion, int frame)
 {
 	auto lump =  fileSystem.ReadFile (SourceLump);
-	auto source = lump.GetBytes();
+	auto source = lump.bytes();
 
 	auto Work = (uint32_t*)bmp->GetPixels();
 	for(int i = 0; i < Width * Height / 2; i++)
@@ -339,7 +339,7 @@ FStrifeStartupTexture::FStrifeStartupTexture (int lumpnum, int w, int h)
 PalettedPixels FStrifeStartupTexture::CreatePalettedPixels(int conversion, int frame)
 {
 	auto lump =  fileSystem.ReadFile (SourceLump);
-	auto source = lump.GetBytes();
+	auto source = lump.bytes();
 	PalettedPixels Pixels(Width*Height);
 	const uint8_t *remap = ImageHelpers::GetRemap(conversion == luminance);
 	ImageHelpers::FlipNonSquareBlockRemap(Pixels.Data(), source, Width, Height, Width, remap);

source/common/textures/formats/webptexture.cpp

@@ -142,7 +142,7 @@ int FWebPTexture::CopyPixels(FBitmap *bmp, int conversion, int frame)
 	config.output.u.RGBA.stride = bmp->GetPitch();
 	config.output.is_external_memory = 1;
 
-	(void)WebPDecode(bytes.GetBytes(), bytes.GetSize(), &config);
+	(void)WebPDecode(bytes.bytes(), bytes.size(), &config);
 
 	return 0;
 }

source/common/textures/multipatchtexturebuilder.cpp

@@ -398,12 +398,12 @@ void FMultipatchTextureBuilder::AddTexturesLumps(int lump1, int lump2, int patch
 	if (lump1 >= 0)
 	{
 		auto texdir = fileSystem.ReadFile(lump1);
-		AddTexturesLump(texdir.GetMem(), fileSystem.FileLength(lump1), lump1, patcheslump, firstdup, true);
+		AddTexturesLump(texdir.data(), fileSystem.FileLength(lump1), lump1, patcheslump, firstdup, true);
 	}
 	if (lump2 >= 0)
 	{
 		auto texdir = fileSystem.ReadFile(lump2);
-		AddTexturesLump(texdir.GetMem(), fileSystem.FileLength(lump2), lump2, patcheslump, firstdup, false);
+		AddTexturesLump(texdir.data(), fileSystem.FileLength(lump2), lump2, patcheslump, firstdup, false);
 	}
 }
 

source/common/textures/texturemanager.cpp

@@ -585,7 +585,7 @@ void FTextureManager::AddGroup(int wadnum, int ns, ETextureType usetype)
 				}
 				progressFunc();
 			}
-			else if (ns == ns_flats && fileSystem.GetFileFlags(firsttx) & LUMPF_MAYBEFLAT)
+			else if (ns == ns_flats && fileSystem.GetFileFlags(firsttx) & RESFF_MAYBEFLAT)
 			{
 				if (fileSystem.CheckNumForName(Name, ns) < firsttx)
 				{
@@ -971,7 +971,7 @@ void FTextureManager::AddTexturesForWad(int wadnum, FMultipatchTextureBuilder &b
 		if (ns == ns_global)
 		{
 			// In Zips all graphics must be in a separate namespace.
-			if (fileSystem.GetFileFlags(i) & LUMPF_FULLPATH) continue;
+			if (fileSystem.GetFileFlags(i) & RESFF_FULLPATH) continue;
 
 			// Ignore lumps with empty names.
 			if (fileSystem.CheckFileName(i, "")) continue;
@@ -1425,7 +1425,7 @@ int FTextureManager::GuesstimateNumTextures ()
 			break;
 
 		default:
-			if (fileSystem.GetFileFlags(i) & LUMPF_MAYBEFLAT) numtex++;
+			if (fileSystem.GetFileFlags(i) & RESFF_MAYBEFLAT) numtex++;
 
 			break;
 		}

source/common/utility/cmdlib.cpp

@@ -1045,7 +1045,7 @@ void uppercopy(char* to, const char* from)
 FString GetStringFromLump(int lump, bool zerotruncate)
 {
 	auto fd = fileSystem.ReadFile(lump);
-	FString ScriptBuffer(fd.GetString(), fd.GetSize());
+	FString ScriptBuffer(fd.string(), fd.size());
 	if (zerotruncate) ScriptBuffer.Truncate(strlen(ScriptBuffer.GetChars()));	// this is necessary to properly truncate the generated string to not contain 0 bytes.
 	return ScriptBuffer;
 }

source/common/utility/palette.cpp

@@ -681,8 +681,8 @@ FString V_GetColorStringByName(const char* name, FScriptPosition* sc)
 	}
 
 	auto rgbNames = fileSystem.ReadFile(rgblump);
-	rgb = rgbNames.GetString();
+	rgb = rgbNames.string();
-	rgbEnd = rgb + rgbNames.GetSize();
+	rgbEnd = rgb + rgbNames.size();
 	step = 0;
 	namelen = strlen(name);
 
@@ -930,11 +930,11 @@ int ReadPalette(int lumpnum, uint8_t* buffer)
 		return 0;
 	}
 	auto lump =  fileSystem.ReadFile(lumpnum);
-	auto lumpmem = lump.GetBytes();
+	auto lumpmem = lump.bytes();
 	memset(buffer, 0, 768);
 
 	FileReader fr;
-	fr.OpenMemory(lumpmem, lump.GetSize());
+	fr.OpenMemory(lumpmem, lump.size());
 	auto png = M_VerifyPNG(fr);
 	if (png)
 	{
@@ -964,7 +964,7 @@ int ReadPalette(int lumpnum, uint8_t* buffer)
 	{
 		FScanner sc;
 
-		sc.OpenMem(fileSystem.GetFileFullName(lumpnum), (char*)lumpmem, int(lump.GetSize()));
+		sc.OpenMem(fileSystem.GetFileFullName(lumpnum), (char*)lumpmem, int(lump.size()));
 		sc.MustGetString();
 		sc.MustGetNumber();	// version - ignore
 		sc.MustGetNumber();
@@ -982,7 +982,7 @@ int ReadPalette(int lumpnum, uint8_t* buffer)
 	}
 	else
 	{
-		memcpy(buffer, lumpmem, min<size_t>(768, lump.GetSize()));
+		memcpy(buffer, lumpmem, min<size_t>(768, lump.size()));
 		return 256;
 	}
 }

source/common/utility/writezip.cpp

@@ -0,0 +1,239 @@
+/*
+** writezip.cpp
+**
+**---------------------------------------------------------------------------
+** Copyright 1998-2009 Randy Heit
+** Copyright 2005-2023 Christoph Oelckers
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+**
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**---------------------------------------------------------------------------
+**
+**
+*/
+
+#include <stdint.h>
+#include <algorithm>
+#include "tarray.h"
+#include "files.h"
+#include "m_swap.h"
+#include "w_zip.h"
+
+using FileSys::FCompressedBuffer;
+
+
+
+//==========================================================================
+//
+// time_to_dos
+//
+// Converts time from struct tm to the DOS format used by zip files.
+//
+//==========================================================================
+
+static std::pair<uint16_t, uint16_t> time_to_dos(struct tm *time)
+{
+	std::pair<uint16_t, uint16_t> val;
+	if (time == NULL || time->tm_year < 80)
+	{
+		val.first = val.second = 0;
+	}
+	else
+	{
+		val.first = (time->tm_year - 80) * 512 + (time->tm_mon + 1) * 32 + time->tm_mday;
+		val.second= time->tm_hour * 2048 + time->tm_min * 32 + time->tm_sec / 2;
+	}
+	return val;
+}
+
+//==========================================================================
+//
+// append_to_zip
+//
+// Write a given file to the zipFile.
+// 
+// zipfile: zip object to be written to
+// 
+// returns: position = success, -1 = error
+//
+//==========================================================================
+
+static int AppendToZip(FileWriter *zip_file, const FCompressedBuffer &content, std::pair<uint16_t, uint16_t> &dostime)
+{
+	FZipLocalFileHeader local;
+	int position;
+
+	int flags = 0;
+	int method = content.mMethod;
+	if (method >= FileSys::METHOD_IMPLODE_MIN && method <= FileSys::METHOD_IMPLODE_MAX)
+	{
+		flags = method - FileSys::METHOD_IMPLODE_MIN;
+		method = FileSys::METHOD_IMPLODE;
+	}
+	else if (method == FileSys::METHOD_DEFLATE)
+	{
+		flags = 2;
+	}
+	else if (method >= 1337)
+		return -1;
+
+	local.Magic = ZIP_LOCALFILE;
+	local.VersionToExtract[0] = 20;
+	local.VersionToExtract[1] = 0;
+	local.Flags = LittleShort((uint16_t)flags);
+	local.Method = LittleShort((uint16_t)method);
+	local.ModDate = LittleShort(dostime.first);
+	local.ModTime = LittleShort(dostime.second);
+	local.CRC32 = content.mCRC32;
+	local.UncompressedSize = LittleLong((unsigned)content.mSize);
+	local.CompressedSize = LittleLong((unsigned)content.mCompressedSize);
+	local.NameLength = LittleShort((unsigned short)strlen(content.filename));
+	local.ExtraLength = 0;
+
+	// Fill in local directory header.
+
+	position = (int)zip_file->Tell();
+
+	// Write out the header, file name, and file data.
+	if (zip_file->Write(&local, sizeof(local)) != sizeof(local) ||
+		zip_file->Write(content.filename, strlen(content.filename)) != strlen(content.filename) ||
+		zip_file->Write(content.mBuffer, content.mCompressedSize) != content.mCompressedSize)
+	{
+		return -1;
+	}
+	return position;
+}
+
+
+//==========================================================================
+//
+// write_central_dir
+//
+// Writes the central directory entry for a file.
+//
+//==========================================================================
+
+int AppendCentralDirectory(FileWriter *zip_file, const FCompressedBuffer &content, std::pair<uint16_t, uint16_t> &dostime, int position)
+{
+	FZipCentralDirectoryInfo dir;
+
+	int flags = 0;
+	int method = content.mMethod;
+	if (method >= FileSys::METHOD_IMPLODE_MIN && method <= FileSys::METHOD_IMPLODE_MAX)
+	{
+		flags = method - FileSys::METHOD_IMPLODE_MIN;
+		method = FileSys::METHOD_IMPLODE;
+	}
+	else if (method == FileSys::METHOD_DEFLATE)
+	{
+		flags = 2;
+	}
+	else if (method >= 1337)
+		return -1;
+
+	dir.Magic = ZIP_CENTRALFILE;
+	dir.VersionMadeBy[0] = 20;
+	dir.VersionMadeBy[1] = 0;
+	dir.VersionToExtract[0] = 20;
+	dir.VersionToExtract[1] = 0;
+	dir.Flags = LittleShort((uint16_t)flags);
+	dir.Method = LittleShort((uint16_t)method);
+	dir.ModTime = LittleShort(dostime.first);
+	dir.ModDate = LittleShort(dostime.second);
+	dir.CRC32 = content.mCRC32;
+	dir.CompressedSize32 = LittleLong((unsigned)content.mCompressedSize);
+	dir.UncompressedSize32 = LittleLong((unsigned)content.mSize);
+	dir.NameLength = LittleShort((unsigned short)strlen(content.filename));
+	dir.ExtraLength = 0;
+	dir.CommentLength = 0;
+	dir.StartingDiskNumber = 0;
+	dir.InternalAttributes = 0;
+	dir.ExternalAttributes = 0;
+	dir.LocalHeaderOffset32 = LittleLong((unsigned)position);
+
+	if (zip_file->Write(&dir, sizeof(dir)) != sizeof(dir) ||
+		zip_file->Write(content.filename,  strlen(content.filename)) != strlen(content.filename))
+	{
+		return -1;
+	}
+	return 0;
+}
+
+bool WriteZip(const char* filename, const FCompressedBuffer* content, size_t contentcount)
+{
+	// try to determine local time
+	struct tm *ltime;
+	time_t ttime;
+	ttime = time(nullptr);
+	ltime = localtime(&ttime);
+	auto dostime = time_to_dos(ltime);
+
+	TArray<int> positions;
+
+	auto f = FileWriter::Open(filename);
+	if (f != nullptr)
+	{
+		for (size_t i = 0; i < contentcount; i++)
+		{
+			int pos = AppendToZip(f, content[i], dostime);
+			if (pos == -1)
+			{
+				delete f;
+				remove(filename);
+				return false;
+			}
+			positions.Push(pos);
+		}
+
+		int dirofs = (int)f->Tell();
+		for (size_t i = 0; i < contentcount; i++)
+		{
+			if (AppendCentralDirectory(f, content[i], dostime, positions[i]) < 0)
+			{
+				delete f;
+				remove(filename);
+				return false;
+			}
+		}
+
+		// Write the directory terminator.
+		FZipEndOfCentralDirectory dirend;
+		dirend.Magic = ZIP_ENDOFDIR;
+		dirend.DiskNumber = 0;
+		dirend.FirstDisk = 0;
+		dirend.NumEntriesOnAllDisks = dirend.NumEntries = LittleShort((uint16_t)contentcount);
+		dirend.DirectoryOffset = LittleLong((unsigned)dirofs);
+		dirend.DirectorySize = LittleLong((uint32_t)(f->Tell() - dirofs));
+		dirend.ZipCommentLength = 0;
+		if (f->Write(&dirend, sizeof(dirend)) != sizeof(dirend))
+		{
+			delete f;
+			remove(filename);
+			return false;
+		}
+		delete f;
+		return true;
+	}
+	return false;
+}

source/core/initfs.cpp

@@ -433,10 +433,9 @@ void InitFileSystem(TArray<GrpEntry>& groups)
 		lfi.embeddings = { "blood.rff", "sounds.rff" };
 	}
 
-	lfi.dotFilter = LumpFilter.GetChars();
-
 	if (isDukeEngine()) lfi.gameTypeFilter.push_back("DukeEngine");
 	if (isDukeLike()) lfi.gameTypeFilter.push_back("DukeLike");
+	lfi.gameTypeFilter.push_back(LumpFilter.GetChars());
 
 	lfi.postprocessFunc = [&]()
 	{
@@ -448,8 +447,8 @@ void InitFileSystem(TArray<GrpEntry>& groups)
 		FILE* f = fopen("filesystem.dir", "wb");
 		for (int num = 0; num < fileSystem.GetNumEntries(); num++)
 		{
-			auto fd = fileSystem.FileLength(num);
+			int64_t fd = fileSystem.FileLength(num);
-			fprintf(f, "%.50s   %60s  %d\n", fileSystem.GetFileFullName(num), fileSystem.GetResourceFileFullName(fileSystem.GetFileContainer(num)), fd);
+			fprintf(f, "%.50s   %60s  %lld\n", fileSystem.GetFileFullName(num), fileSystem.GetResourceFileFullName(fileSystem.GetFileContainer(num)), fd);
 		}
 		fclose(f);
 	}

source/core/savegamehelp.cpp

@@ -69,6 +69,7 @@
 
 #include "buildtiles.h"
 #include "fs_findfile.h"
+#include "resourcefile.h"
 
 
 
@@ -79,6 +80,7 @@ extern FString BackupSaveGame;
 int SaveVersion;
 
 void SerializeMap(FSerializer &arc);
+bool WriteZip(const char* filename, const FileSys::FCompressedBuffer* content, size_t contentcount);
 
 BEGIN_BLD_NS
 FSerializer& Serialize(FSerializer& arc, const char* keyname, XWALL& w, XWALL* def);
@@ -132,7 +134,7 @@ static void SerializeSession(FSerializer& arc)
 
 bool ReadSavegame(const char* name)
 {
-	auto savereader = FResourceFile::OpenResourceFile(name, true);
+	auto savereader = FileSys::FResourceFile::OpenResourceFile(name, true);
 
 	if (savereader != nullptr)
 	{
@@ -245,7 +247,7 @@ bool WriteSavegame(const char* filename, const char *name)
 	M_FinishPNG(&savepic);
 
 	auto picdata = savepic.GetBuffer();
-	FileSys::FCompressedBuffer bufpng = { picdata->size(), picdata->size(), FileSys::METHOD_STORED, 0, static_cast<unsigned int>(crc32(0, &(*picdata)[0], picdata->size())), (char*)&(*picdata)[0] };
+	FileSys::FCompressedBuffer bufpng = { picdata->size(), picdata->size(), FileSys::METHOD_STORED, static_cast<unsigned int>(crc32(0, &(*picdata)[0], picdata->size())), (char*)&(*picdata)[0] };
 
 	TArray<FileSys::FCompressedBuffer> savegame_content;
 	TArray<FString> savegame_filenames;

source/games/blood/src/blood.cpp

@@ -560,8 +560,8 @@ void GameInterface::loadPalette(void)
 	for (int i = 0; i < 5; i++)
 	{
 		auto pal = fileSystem.ReadFileFullName(PAL[i]);
-		if (pal.GetSize() < 768) I_FatalError("%s: file too small", PAL[i]);
+		if (pal.size() < 768) I_FatalError("%s: file too small", PAL[i]);
-		paletteSetColorTable(i, (const uint8_t*)pal.GetMem(), false, false);
+		paletteSetColorTable(i, pal.bytes(), false, false);
 	}
 
 	numshades = 64;
@@ -574,12 +574,12 @@ void GameInterface::loadPalette(void)
 			else continue;
 		}
 		auto data = fileSystem.ReadFile(lump);
-		if (data.GetSize() != 64 * 256)
+		if (data.size() != 64 * 256)
 		{
 			if (i < 15) I_FatalError("%s: Incorrect PLU size", PLU[i]);
 			else continue;
 		}
-		lookups.setTable(i, (const uint8_t*)data.GetMem());
+		lookups.setTable(i, data.bytes());
 	}
 
 	lookups.setFadeColor(1, 255, 255, 255);

source/games/blood/src/sound.cpp

@@ -76,7 +76,7 @@ static void S_AddBloodSFX(int lumpnum)
 	}
 
 	auto sfxlump = fileSystem.ReadFile(lumpnum);
-	SFX* sfx = (SFX*)sfxlump.GetMem();
+	SFX* sfx = (SFX*)sfxlump.data();
 	ByteSwapSFX(sfx);
 
 	FStringf rawname("%s.raw", sfx->rawName);

source/games/exhumed/src/sound.cpp

@@ -177,7 +177,7 @@ int LoadSound(const char* name)
     {
         auto check = fileSystem.ReadFile(lump);
 		bool loops = false;
-        if (check.GetSize() > 26 && check.GetString()[26] == 6 && !memcmp("Creative Voice File", check.GetString(), 19))
+        if (check.size() > 26 && check.string()[26] == 6 && !memcmp("Creative Voice File", check.string(), 19))
         {
             // This game uses the actual loop point information in the sound data as its only means to check if a sound is looped.
             loops = true;

source/games/sw/src/scrip2.cpp

@@ -78,7 +78,7 @@ TArray<uint8_t> LoadScriptFile(const char *filename)
 		return ret;
 	}
 
-    ret.Resize(fp.GetLength() + 1);
+    ret.Resize((unsigned)fp.GetLength() + 1);
     if (fp.Read(ret.Data(), fp.GetLength()) < fp.GetLength())
     {
         scriptline = 1;