set up the new Entries array.

src/common/filesystem/include/fs_files.h

@@ -85,6 +85,13 @@ enum
 	METHOD_PPMD = 98,
 	METHOD_LZSS = 1337,	// not used in Zips - this is for Console Doom compression
 	METHOD_ZLIB = 1338,	// Zlib stream with header, used by compressed nodes.
+	METHOD_IMPLODE_MIN = 1000, // having discrete types for these avoids keeping around the GPFlags word in Zips.
+	METHOD_IMPLODE_0 = 1000,
+	METHOD_IMPLODE_2 = 1002,
+	METHOD_IMPLODE_4 = 1004,
+	METHOD_IMPLODE_6 = 1006,
+	METHOD_IMPLODE_MAX = 1006,
+	METHOD_INVALID = 0x7fff,
 	METHOD_TRANSFEROWNER = 0x8000,
 };
 

src/common/filesystem/include/resourcefile.h

@@ -82,6 +82,14 @@ enum ELumpFlags
 	LUMPF_EMBEDDED = 4,		// marks an embedded resource file for later processing.
 	LUMPF_SHORTNAME = 8,	// the stored name is a short extension-less name
 	LUMPF_COMPRESSED = 16,	// compressed or encrypted, i.e. cannot be read with the container file's reader.
+	
+	RESFF_MAYBEFLAT = 1,	// might be a flat inside a WAD outside F_START/END
+	RESFF_FULLPATH = 2,		// contains a full path. This will trigger extended namespace checks when looking up short names.
+	RESFF_EMBEDDED = 4,		// marks an embedded resource file for later processing.
+	RESFF_SHORTNAME = 8,	// the stored name is a short extension-less name
+	RESFF_COMPRESSED = 16,	// compressed or encrypted, i.e. cannot be read with the container file's reader.
+	RESFF_NEEDFILESTART = 32,	// The real position is not known yet and needs to be calculated on access
+	
 };
 
 // This holds a compresed Zip entry with all needed info to decompress it.
@@ -172,12 +180,26 @@ protected:
 
 };
 
+struct FResourceEntry
+{
+	size_t Length;
+	size_t CompressedSize;
+	const char* FileName;
+	size_t Position;
+	int ResourceID;
+	uint32_t CRC32;
+	uint16_t Flags;
+	uint16_t Method;
+	int16_t Namespace;
+};
+
 class FResourceFile
 {
 public:
 protected:
 	FileReader Reader;
 	const char* FileName;
+	FResourceEntry* Entries = nullptr;
 	uint32_t NumLumps;
 	char Hash[48];
 	StringPool* stringpool;
@@ -185,6 +207,9 @@ protected:
 	FResourceFile(const char *filename, StringPool* sp);
 	FResourceFile(const char *filename, FileReader &r, StringPool* sp);
 
+	const char* NormalizeFileName(const char* fn, int fallbackcp = 0);
+	void AllocateEntries(int count);
+
 	// for archives that can contain directories
 	void GenerateHash();
 	void PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterInfo *filter);
@@ -214,15 +239,13 @@ public:
 	int EntryCount() const { return NumLumps; }
 	int FindEntry(const char* name);
 
-	size_t Length(int entry)
+	size_t Length(uint32_t entry)
 	{
-		auto l = GetLump(entry);
+		return (entry < NumLumps) ? Entries[entry].Length : 0;
-		return l ? l->LumpSize : -1;
 	}
-	size_t Offset(int entry)
+	size_t Offset(uint32_t entry)
 	{
-		auto l = GetLump(entry);
+		return (entry < NumLumps) ? Entries[entry].Position : 0;
-		return l ? l->GetFileOffset() : -1;
 	}
 
 	FileReader GetEntryReader(int entry, bool newreader = true)
@@ -231,22 +254,24 @@ public:
 		return l ? l->NewReader() : FileReader();
 	}
 
-	int GetEntryFlags(int entry)
+	int GetEntryFlags(uint32_t entry)
 	{
-		auto l = GetLump(entry);
+		return (entry < NumLumps) ? Entries[entry].Flags : 0;
-		return l ? l->Flags : 0;
 	}
 
-	int GetEntryNamespace(int entry)
+	int GetEntryNamespace(uint32_t entry)
 	{
-		auto l = GetLump(entry);
+		return (entry < NumLumps) ? Entries[entry].Namespace : ns_hidden;
-		return l ? l->GetNamespace() : 0;
 	}
 
-	int GetEntryResourceID(int entry)
+	int GetEntryResourceID(uint32_t entry)
 	{
-		auto l = GetLump(entry);
+		return (entry < NumLumps) ? Entries[entry].ResourceID : -1;
-		return l ? l->GetIndexNum() : 0;
+	}
+
+	const char* getName(uint32_t entry)
+	{
+		return (entry < NumLumps) ? Entries[entry].FileName : nullptr;
 	}
 
 	FileData Read(int entry)
@@ -255,11 +280,6 @@ public:
 		return fr.Read();
 	}
 
-	const char* getName(int entry)
-	{
-		auto l = GetLump(entry);
-		return l ? l->FullName : nullptr;
-	}
 	FCompressedBuffer GetRawData(int entry)
 	{
 		auto l = GetLump(entry);

src/common/filesystem/source/file_7z.cpp

@@ -38,6 +38,7 @@
 #include "7zCrc.h"
 #include "resourcefile.h"
 #include "fs_findfile.h"
+#include "unicode.h"
 
 
 namespace FileSys {
@@ -247,12 +248,13 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
 
 	CSzArEx* const archPtr = &Archive->DB;
 
+	AllocateEntries(archPtr->NumFiles);
 	NumLumps = archPtr->NumFiles;
 	Lumps = new F7ZLump[NumLumps];
 
 	F7ZLump *lump_p = Lumps;
 	std::u16string nameUTF16;
-	std::string nameASCII;
+	std::vector<char> nameASCII;
 
 	for (uint32_t i = 0; i < NumLumps; ++i)
 	{
@@ -273,15 +275,19 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
 
 		nameUTF16.resize((unsigned)nameLength);
 		nameASCII.resize((unsigned)nameLength);
-		// note that the file system is not equipped to handle non-ASCII, so don't bother with proper Unicode conversion here.
-		SzArEx_GetFileNameUtf16(archPtr, i, (UInt16*)nameUTF16.data());
-		for (size_t c = 0; c < nameLength; ++c)
-		{
-			nameASCII[c] = tolower(static_cast<char>(nameUTF16[c]));
-		}
-		FixPathSeparator(&nameASCII.front());
 
-		lump_p->LumpNameSetup(nameASCII.c_str(), stringpool);
+		SzArEx_GetFileNameUtf16(archPtr, i, (UInt16*)nameUTF16.data());
+		utf16_to_utf8((uint16_t*)nameUTF16.data(), nameASCII);
+
+		Entries[i].FileName = NormalizeFileName(nameASCII.data());
+		Entries[i].Length = SzArEx_GetFileSize(archPtr, i);
+		Entries[i].Flags = RESFF_FULLPATH|RESFF_COMPRESSED;
+		Entries[i].ResourceID = -1;
+		Entries[i].Namespace = ns_global;
+		Entries[i].Method = METHOD_INVALID;
+		Entries[i].Position = i;
+
+		lump_p->LumpNameSetup(nameASCII.data(), stringpool);
 		lump_p->LumpSize = static_cast<int>(SzArEx_GetFileSize(archPtr, i));
 		lump_p->Owner = this;
 		lump_p->Flags = LUMPF_FULLPATH|LUMPF_COMPRESSED;

src/common/filesystem/source/file_directory.cpp

@@ -56,10 +56,10 @@ std::wstring toWide(const char* str);
 
 struct FDirectoryLump : public FResourceLump
 {
+	const char* mBasePath;
 	FileReader NewReader() override;
 	int FillCache() override;
 
-	const char* mFullPath;
 };
 
 
@@ -73,9 +73,10 @@ class FDirectory : public FResourceFile
 {
 	TArray<FDirectoryLump> Lumps;
 	const bool nosubdir;
+	const char* mBasePath;
 
 	int AddDirectory(const char* dirpath, LumpFilterInfo* filter, FileSystemMessageFunc Printf);
-	void AddEntry(const char *fullpath, const char* relpath, int size);
+	void AddEntry(const char* relpath, int size);
 
 public:
 	FDirectory(const char * dirname, StringPool* sp, bool nosubdirflag = false);
@@ -116,8 +117,17 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy
 	}
 	else
 	{
+		mBasePath = nullptr;
+		AllocateEntries(list.size());
 		for(auto& entry : list)
 		{
+			if (mBasePath == nullptr)
+			{
+				// extract the base path from the first entry to cover changes made in ScanDirectory.
+				auto full = entry.FilePath.find(entry.FilePathRel);
+				std::string path(entry.FilePath, full);
+				mBasePath = stringpool->Strdup(path.c_str());
+			}
 			if (!entry.isDirectory)
 			{
 				auto fi = entry.FileName;
@@ -128,6 +138,7 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy
 					continue;
 				}
 
+
 				if (filter->filenamecheck == nullptr || filter->filenamecheck(fi.c_str(), entry.FilePath.c_str()))
 				{
 					if (entry.Length > 0x7fffffff)
@@ -135,7 +146,17 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy
 						Printf(FSMessageLevel::Warning, "%s is larger than 2GB and will be ignored\n", entry.FilePath.c_str());
 						continue;
 					}
-					AddEntry(entry.FilePath.c_str(), entry.FilePathRel.c_str(), (int)entry.Length);
+					AddEntry(entry.FilePathRel.c_str(), (int)entry.Length);
+					int index = count;
+					// for internal access we use the normalized form of the relative path.
+					Entries[index].FileName = NormalizeFileName(entry.FilePathRel.c_str());
+					Entries[index].Length = entry.Length;
+					Entries[index].Flags = RESFF_FULLPATH;
+					Entries[index].ResourceID = -1;
+					Entries[index].Method = METHOD_STORED;
+					Entries[index].Namespace = ns_global;
+					index++;
+
 					count++;
 				}
 			}
@@ -163,13 +184,10 @@ bool FDirectory::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 //
 //==========================================================================
 
-void FDirectory::AddEntry(const char *fullpath, const char* relpath, int size)
+void FDirectory::AddEntry(const char* relpath, int size)
 {
 	FDirectoryLump *lump_p = &Lumps[Lumps.Reserve(1)];
 
-	// Store the full path here so that we can access the file later, even if it is from a filter directory.
-	lump_p->mFullPath = stringpool->Strdup(fullpath);
-
 	// [mxd] Convert name to lowercase
 	std::string name = relpath;
 	for (auto& c : name) c = tolower(c);
@@ -192,7 +210,8 @@ void FDirectory::AddEntry(const char *fullpath, const char* relpath, int size)
 FileReader FDirectoryLump::NewReader()
 {
 	FileReader fr;
-	fr.OpenFile(mFullPath);
+	std::string fn = mBasePath; fn += this->FullName;
+	fr.OpenFile(fn.c_str());
 	return fr;
 }
 
@@ -206,7 +225,8 @@ int FDirectoryLump::FillCache()
 {
 	FileReader fr;
 	Cache = new char[LumpSize];
-	if (!fr.OpenFile(mFullPath))
+	std::string fn = mBasePath; fn += this->FullName;
+	if (!fr.OpenFile(fn.c_str()))
 	{
 		throw FileSystemException("unable to open file");
 	}

src/common/filesystem/source/file_grp.cpp

@@ -101,6 +101,7 @@ bool FGrpFile::Open(LumpFilterInfo* filter)
 	GrpHeader header;
 
 	Reader.Read(&header, sizeof(header));
+	AllocateEntries(header.NumLumps);
 	NumLumps = LittleLong(header.NumLumps);
 
 	GrpLump *fileinfo = new GrpLump[NumLumps];
@@ -112,12 +113,21 @@ bool FGrpFile::Open(LumpFilterInfo* filter)
 
 	for(uint32_t i = 0; i < NumLumps; i++)
 	{
+		Entries[i].Position = Position;
+		Entries[i].Length = LittleLong(fileinfo[i].Size);
+		Position += fileinfo[i].Size;
+		Entries[i].Flags = 0;
+		Entries[i].Namespace = ns_global;
+		fileinfo[i].NameWithZero[12] = '\0';	// Be sure filename is null-terminated
+		Entries[i].ResourceID = -1;
+		Entries[i].Method = METHOD_STORED;
+		Entries[i].FileName = NormalizeFileName(fileinfo[i].Name);
+	
 		Lumps[i].Owner = this;
 		Lumps[i].Position = Position;
-		Lumps[i].LumpSize = LittleLong(fileinfo[i].Size);
+		Lumps[i].LumpSize = Entries[i].Length;
 		Position += fileinfo[i].Size;
 		Lumps[i].Flags = 0;
-		fileinfo[i].NameWithZero[12] = '\0';	// Be sure filename is null-terminated
 		Lumps[i].LumpNameSetup(fileinfo[i].NameWithZero, stringpool);
 	}
 	GenerateHash();

src/common/filesystem/source/file_lump.cpp

@@ -68,6 +68,15 @@ FLumpFile::FLumpFile(const char *filename, FileReader &file, StringPool* sp)
 
 bool FLumpFile::Open(LumpFilterInfo*)
 {
+	AllocateEntries(1);
+	Entries[0].FileName = NormalizeFileName(ExtractBaseName(FileName, true).c_str());
+	Entries[0].Namespace = ns_global;
+	Entries[0].ResourceID = -1;
+	Entries[0].Position = 0;
+	Entries[0].Length = Reader.GetLength();
+	Entries[0].Method = METHOD_STORED;
+	Entries[0].Flags = 0;
+
 	Lumps.Resize(1);
 	Lumps[0].LumpNameSetup(ExtractBaseName(FileName, true).c_str(), stringpool);
 	Lumps[0].Owner = this;

src/common/filesystem/source/file_pak.cpp

@@ -95,6 +95,7 @@ bool FPakFile::Open(LumpFilterInfo* filter)
 	dpackheader_t header;
 
 	Reader.Read(&header, sizeof(header));
+	AllocateEntries(header.dirlen / sizeof(dpackfile_t));
 	NumLumps = LittleLong(header.dirlen) / sizeof(dpackfile_t);
 	header.dirofs = LittleLong(header.dirofs);
 
@@ -106,6 +107,14 @@ bool FPakFile::Open(LumpFilterInfo* filter)
 
 	for(uint32_t i = 0; i < NumLumps; i++)
 	{
+		Entries[i].Position = LittleLong(fileinfo[i].filepos);
+		Entries[i].Length = LittleLong(fileinfo[i].filelen);
+		Entries[i].Flags = RESFF_FULLPATH;
+		Entries[i].Namespace = ns_global;
+		Entries[i].ResourceID = -1;
+		Entries[i].Method = METHOD_STORED;
+		Entries[i].FileName = NormalizeFileName(fileinfo[i].name);
+	
 		Lumps[i].LumpNameSetup(fileinfo[i].name, stringpool);
 		Lumps[i].Flags = LUMPF_FULLPATH;
 		Lumps[i].Owner = this;

src/common/filesystem/source/file_rff.cpp

@@ -143,6 +143,7 @@ bool FRFFFile::Open(LumpFilterInfo*)
 
 	Reader.Read(&header, sizeof(header));
 
+	AllocateEntries(LittleLong(header.NumLumps));
 	NumLumps = LittleLong(header.NumLumps);
 	header.DirOfs = LittleLong(header.DirOfs);
 	lumps = new RFFLump[header.NumLumps];
@@ -154,6 +155,23 @@ bool FRFFFile::Open(LumpFilterInfo*)
 
 	for (uint32_t i = 0; i < NumLumps; ++i)
 	{
+		Entries[i].Position = LittleLong(lumps[i].FilePos);
+		Entries[i].Length = LittleLong(lumps[i].Size);
+		Entries[i].Flags = 0;
+		Entries[i].Method = METHOD_STORED;
+		if (lumps[i].Flags & 0x10)
+		{
+			Entries[i].Flags = RESFF_COMPRESSED;	// flags the lump as not directly usable
+			Entries[i].Method = METHOD_INVALID;
+		}
+		else
+		{
+			Entries[i].Flags = 0;
+			Entries[i].Method = METHOD_STORED;
+		}
+		Entries[i].Namespace = ns_global;
+		Entries[i].ResourceID = LittleLong(lumps[i].IndexNum);
+	
 		Lumps[i].Position = LittleLong(lumps[i].FilePos);
 		Lumps[i].LumpSize = LittleLong(lumps[i].Size);
 		Lumps[i].Owner = this;
@@ -174,6 +192,7 @@ bool FRFFFile::Open(LumpFilterInfo*)
 		name[len+3] = lumps[i].Extension[2];
 		name[len+4] = 0;
 		Lumps[i].LumpNameSetup(name, stringpool);
+		Entries[i].FileName = NormalizeFileName(name);
 	}
 	delete[] lumps;
 	GenerateHash();

src/common/filesystem/source/file_ssi.cpp

@@ -70,6 +70,7 @@ FSSIFile::FSSIFile(const char *filename, FileReader &file, StringPool* sp)
 
 bool FSSIFile::Open(int version, int EntryCount, LumpFilterInfo*)
 {
+	AllocateEntries(EntryCount*2);
 	NumLumps = EntryCount*2;
 	Lumps.Resize(EntryCount*2);
 
@@ -85,7 +86,30 @@ bool FSSIFile::Open(int version, int EntryCount, LumpFilterInfo*)
 		fn[strlength] = 0;
 		int flength = Reader.ReadInt32();
 
+		Entries[i].Position = j;
+		Entries[i].Length = flength;
+		Entries[i].Flags = 0;
+		Entries[i].Namespace = ns_global;
+		Entries[i].Method = METHOD_STORED;
+		Entries[i].ResourceID = -1;
+		Entries[i].FileName = NormalizeFileName(fn);
+		if (strstr(fn, ".GRP")) Entries[i].Flags |= RESFF_EMBEDDED;
 
+		// SSI files can swap the order of the extension's characters - but there's no reliable detection for this and it can be mixed inside the same container, 
+		// so we have no choice but to create another file record for the altered name.
+		std::swap(fn[strlength - 1], fn[strlength - 3]);
+
+		Entries[i + 1].Position = j;
+		Entries[i + 1].Length = flength;
+		Entries[i + 1].Flags = 0;
+		Entries[i + 1].Namespace = ns_global;
+		Entries[i + 1].ResourceID = -1;
+		Entries[i + 1].FileName = NormalizeFileName(fn);
+		Entries[i + 1].Method = METHOD_STORED;
+		if (strstr(fn, ".GRP")) Entries[i + 1].Flags |= RESFF_EMBEDDED;
+
+		// swap back...
+		std::swap(fn[strlength - 1], fn[strlength - 3]);
 		Lumps[i].LumpNameSetup(fn, stringpool);
 		Lumps[i].Position = j;
 		Lumps[i].LumpSize = flength;

src/common/filesystem/source/file_wad.cpp

@@ -37,6 +37,7 @@
 #include "resourcefile.h"
 #include "fs_filesystem.h"
 #include "fs_swap.h"
+#include "fs_stringpool.h"
 
 namespace FileSys {
 	using namespace byteswap;
@@ -192,8 +193,37 @@ bool FWadFile::Open(LumpFilterInfo*, FileSystemMessageFunc Printf)
 	Reader.Seek (InfoTableOfs, FileReader::SeekSet);
 	Reader.Read (fileinfo.Data(), NumLumps * sizeof(wadlump_t));
 
+	AllocateEntries(NumLumps);
 	Lumps.Resize(NumLumps);
 
+	for(uint32_t i = 0; i < NumLumps; i++)
+	{
+		// WAD only supports ASCII. It is also the only format which can use valid backslashes in its names.
+		char n[9];
+		int ishigh = 0;
+		for (int j = 0; j < 8; j++)
+		{
+			if (fileinfo[i].Name[j] & 0x80) ishigh |= 1 << j;
+			n[j] = tolower(fileinfo[i].Name[j]);
+		}
+		n[8] = 0;
+		if (ishigh == 1) n[0] &= 0x7f;
+		else if (ishigh > 1)
+		{
+			// This may not end up printing something proper because we do not know what encoding might have been used.
+			Printf(FSMessageLevel::Warning, "%s: Lump name %s contains invalid characters\n", FileName, Lumps[i].getName());
+		}
+
+		Entries[i].FileName = nullptr;
+		Entries[i].Position = isBigEndian ? BigLong(fileinfo[i].FilePos) : LittleLong(fileinfo[i].FilePos);
+		Entries[i].Length = isBigEndian ? BigLong(fileinfo[i].Size) : LittleLong(fileinfo[i].Size);
+		Entries[i].Namespace = ns_global;
+		Entries[i].Flags = ishigh? RESFF_SHORTNAME | RESFF_COMPRESSED : RESFF_SHORTNAME;
+		Entries[i].Method = ishigh == 1? METHOD_LZSS : METHOD_STORED;
+		Entries[i].FileName = stringpool->Strdup(n);
+		// This doesn't set up the namespace yet.
+	}
+
 	for(uint32_t i = 0; i < NumLumps; i++)
 	{
 		char n[9];
@@ -238,6 +268,11 @@ bool FWadFile::Open(LumpFilterInfo*, FileSystemMessageFunc Printf)
 	SetNamespace("HI_START", "HI_END", ns_hires, Printf);
 	SetNamespace("VX_START", "VX_END", ns_voxels, Printf);
 	SkinHack(Printf);
+
+	for (uint32_t i = 0; i < NumLumps; i++)
+	{
+		Entries[i].Namespace = Lumps[i].Namespace;
+	}
 	return true;
 }
 

src/common/filesystem/source/file_whres.cpp

@@ -84,24 +84,43 @@ bool FWHResFile::Open(LumpFilterInfo*)
 	int nl =1024/3;
 	Lumps.Resize(nl);
 
+	int k;
+	for (k = 0; k < nl; k++)
+	{
+		uint32_t offset = LittleLong(directory[k * 3]) * 4096;
+		uint32_t length = LittleLong(directory[k * 3 + 1]);
+		if (length == 0)
+		{
+			break;
+		}
+	}
+	AllocateEntries(k);
+	NumLumps = k;
 
 	int i = 0;
-	for(int k = 0; k < nl; k++)
+	for(k = 0; k < NumLumps; k++)
 	{
 		uint32_t offset = LittleLong(directory[k*3]) * 4096;
 		uint32_t length = LittleLong(directory[k*3+1]);
-		if (length == 0) break;
 		char num[6];
 		snprintf(num, 6, "/%04d", k);
 		std::string synthname = BaseName;
 		synthname += num;
+		
+		Entries[i].Position = offset;
+		Entries[i].Length = length;
+		Entries[i].Flags = RESFF_FULLPATH;
+		Entries[i].Namespace = ns_global;
+		Entries[i].ResourceID = -1;
+		Entries[i].Method = METHOD_STORED;
+		Entries[i].FileName = NormalizeFileName(synthname.c_str());
+		
 		Lumps[i].LumpNameSetup(synthname.c_str(), stringpool);
 		Lumps[i].Owner = this;
 		Lumps[i].Position = offset;
 		Lumps[i].LumpSize = length;
 		i++;
 	}
-	NumLumps = i;
 	Lumps.Clamp(NumLumps);
 	Lumps.ShrinkToFit();
 	return true;

src/common/filesystem/source/file_zip.cpp

@@ -40,6 +40,7 @@
 #include "ancientzip.h"
 #include "fs_findfile.h"
 #include "fs_swap.h"
+#include "fs_stringpool.h"
 
 namespace FileSys {
 	using namespace byteswap;
@@ -75,7 +76,21 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdi
 		break;
 	}
 
-	// Fixme: These should also use a stream
+	case METHOD_IMPLODE_0:
+	case METHOD_IMPLODE_2:
+	case METHOD_IMPLODE_4:
+	case METHOD_IMPLODE_6:
+	{
+		FZipExploder exploder;
+		if (exploder.Explode((unsigned char*)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize, Method - METHOD_IMPLODE_MIN) == -1)
+		{
+			// decompression failed so zero the cache.
+			memset(Cache, 0, LumpSize);
+		}
+		break;
+	}
+
+	// This can go away once we are done with FResourceLump
 	case METHOD_IMPLODE:
 	{
 		FZipExploder exploder;
@@ -311,6 +326,8 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 
 	dirptr = (char*)directory;
 	lump_p = Lumps;
+	AllocateEntries(NumLumps);
+	auto Entry = Entries;
 	for (uint32_t i = 0; i < NumLumps; i++)
 	{
 		FZipCentralDirectoryInfo *zip_fh = (FZipCentralDirectoryInfo *)dirptr;
@@ -399,6 +416,26 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 			}
 		}
 
+		Entry->FileName = NormalizeFileName(name.c_str());
+		Entry->Length = UncompressedSize;
+		// The start of the Reader will be determined the first time it is accessed.
+		Entry->Flags = RESFF_FULLPATH | RESFF_NEEDFILESTART;
+		Entry->Method = uint8_t(zip_fh->Method);
+		if (Entry->Method != METHOD_STORED) Entry->Flags |= RESFF_COMPRESSED;
+		if (Entry->Method == METHOD_IMPLODE)
+		{
+			// merge the flags into the compression method to tag less data around.
+			if ((zip_fh->Flags & 6) == 2) Entry->Method = METHOD_IMPLODE_2;
+			else if ((zip_fh->Flags & 6) == 4) Entry->Method = METHOD_IMPLODE_4;
+			else if ((zip_fh->Flags & 6) == 6) Entry->Method = METHOD_IMPLODE_6;
+			else Entry->Method = METHOD_IMPLODE_0;
+		}
+		Entry->CRC32 = zip_fh->CRC32;
+		Entry->CompressedSize = CompressedSize;
+		Entry->Position = LocalHeaderOffset;
+
+		Entry++;
+
 		lump_p->LumpNameSetup(name.c_str(), stringpool);
 		lump_p->LumpSize = UncompressedSize;
 		lump_p->Owner = this;
@@ -407,6 +444,13 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 		lump_p->NeedFileStart = true;
 		lump_p->Method = uint8_t(zip_fh->Method);
 		if (lump_p->Method != METHOD_STORED) lump_p->Flags |= LUMPF_COMPRESSED;
+		/*if (lump_p->Method == METHOD_IMPLODE)
+		{
+			// merge the flags into the compression method to tag less data around.
+			if ((zip_fh->Flags & 6) == 2) lump_p->Method = METHOD_IMPLODE_2;
+			if ((zip_fh->Flags & 6) == 4) lump_p->Method = METHOD_IMPLODE_4;
+			if ((zip_fh->Flags & 6) == 6) lump_p->Method = METHOD_IMPLODE_6;
+		}*/
 		lump_p->GPFlags = zip_fh->Flags;
 		lump_p->CRC32 = zip_fh->CRC32;
 		lump_p->CompressedSize = CompressedSize;

src/common/filesystem/source/file_zip.h

@@ -28,7 +28,6 @@ private:
 	FCompressedBuffer GetRawData();
 };
 
-
 //==========================================================================
 //
 // Zip file

src/common/filesystem/source/filesystem.cpp

@@ -101,6 +101,10 @@ struct FileSystem::LumpRecord
 
 	void SetFromLump(FResourceFile* file, int fileindex, int filenum, StringPool* sp, const char* name = nullptr)
 	{
+		if (fileindex == 649 && filenum == 0)
+		{
+			int a = 0;
+		}
 		resfile = file;
 		resindex = fileindex;
 		rfnum = filenum;

src/common/filesystem/source/fs_stringpool.cpp

@@ -104,10 +104,12 @@ StringPool::Block *StringPool::AddBlock(size_t size)
 	return mem;
 }
 
-void *StringPool::iAlloc(size_t size)
+void *StringPool::Alloc(size_t size)
 {
 	Block *block;
 
+	size = ((size)+8) & ~7;
+
 	for (block = TopBlock; block != nullptr; block = block->NextBlock)
 	{
 		void *res = block->Alloc(size);
@@ -122,7 +124,7 @@ void *StringPool::iAlloc(size_t size)
 
 const char* StringPool::Strdup(const char* str)
 {
-	char* p = (char*)iAlloc((strlen(str) + 8) & ~7 );
+	char* p = (char*)Alloc(strlen(str));
 	strcpy(p, str);
 	return p;
 }

src/common/filesystem/source/fs_stringpool.h

@@ -12,12 +12,12 @@ private:
 public:
 	~StringPool();
 	const char* Strdup(const char*);
+	void* Alloc(size_t size);
 
 protected:
 	struct Block;
 
 	Block *AddBlock(size_t size);
-	void *iAlloc(size_t size);
 
 	Block *TopBlock;
 	Block *FreeBlocks;

src/common/filesystem/source/resourcefile.cpp

@@ -39,6 +39,8 @@
 #include "md5.hpp"
 #include "fs_stringpool.h"
 #include "files_internal.h"
+#include "unicode.h"
+#include "fs_findfile.h"
 
 namespace FileSys {
 	
@@ -347,6 +349,59 @@ FResourceFile::~FResourceFile()
 	if (!stringpool->shared) delete stringpool;
 }
 
+//==========================================================================
+//
+// normalize the visible file name in the system
+// to lowercase canonical precomposed Unicode.
+//
+//==========================================================================
+
+const char* FResourceFile::NormalizeFileName(const char* fn, int fallbackcp)
+{
+	if (!fn || !*fn) return "";
+	auto norm = tolower_normalize(fn);
+	if (!norm)
+	{
+		if (fallbackcp == 437)
+		{
+			std::vector<char> buffer;
+			ibm437_to_utf8(fn, buffer);
+			norm = tolower_normalize(buffer.data());
+		}
+		// maybe handle other codepages
+		else
+		{
+			// if the filename is not valid UTF-8, nuke all bytes larger than 0x80 so that we still got something semi-usable
+			std::string ffn = fn;
+			for (auto& c : ffn)
+			{
+				if (c & 0x80) c = '@';
+			}
+			norm = tolower_normalize(&ffn.front());
+		}
+	}
+	FixPathSeparator(norm);
+	auto pooled = stringpool->Strdup(norm);
+	free(norm);
+	return pooled;
+
+}
+
+//==========================================================================
+//
+// allocate the Entries array
+// this also uses the string pool to reduce maintenance
+//
+//==========================================================================
+
+void FResourceFile::AllocateEntries(int count)
+{
+	NumLumps = count;
+	Entries = (FResourceEntry*)stringpool->Alloc(count * sizeof(FResourceEntry));
+}
+
+
+//---------------------------------------------------
 int lumpcmp(const void * a, const void * b)
 {
 	FResourceLump * rec1 = (FResourceLump *)a;
@@ -354,6 +409,14 @@ int lumpcmp(const void * a, const void * b)
 	return stricmp(rec1->getName(), rec2->getName());
 }
 
+int entrycmp(const void* a, const void* b)
+{
+	FResourceEntry* rec1 = (FResourceEntry*)a;
+	FResourceEntry* rec2 = (FResourceEntry*)b;
+	// we are comparing lowercase UTF-8 here 
+	return strcmp(rec1->FileName, rec2->FileName);
+}
+
 //==========================================================================
 //
 // FResourceFile :: GenerateHash
@@ -402,8 +465,12 @@ void FResourceFile::GenerateHash()
 
 void FResourceFile::PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterInfo *filter)
 {
-	// Entries in archives are sorted alphabetically
+	// only do this for archive types which contain full file names. All others are assumed to be pre-sorted.
-	qsort(lumps, NumLumps, lumpsize, lumpcmp);
+	//if (NumLumps < 2 || !(Entries[0].Flags & RESFF_FULLPATH)) return;
+
+	// Entries in archives are sorted alphabetically (for now skip the sorting because both arrays might disagree due to file name postprocessing.)
+	//qsort(lumps, NumLumps, lumpsize, lumpcmp);
+	//qsort(Entries, NumLumps, sizeof(Entries[0]), entrycmp);
 	if (!filter) return;
 
 	// Filter out lumps using the same names as the Autoload.* sections
@@ -460,8 +527,10 @@ int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_
 			FResourceLump *lump = (FResourceLump *)lump_p;
 			assert(strnicmp(lump->FullName, filter.c_str(), filter.length()) == 0);
 			lump->LumpNameSetup(lump->FullName + filter.length(), nullptr);
+			Entries[i].FileName += filter.length();
 		}
 
+#if 0 // disabled until everything is clean again
 		// Move filtered lumps to the end of the lump list.
 		size_t count = (end - start) * lumpsize;
 		void *to = (uint8_t *)lumps + NumLumps * lumpsize - count;
@@ -481,6 +550,7 @@ int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_
 
 			delete[] filteredlumps;
 		}
+#endif
 	}
 	return end - start;
 }