move root folder detection out of file_zip.

added support for Descent Hog and Mvl files., mainly useful for playing Descent's music directly from the asset files.

src/CMakeLists.txt

@@ -1220,6 +1220,8 @@ set( GAME_SOURCES
 	common/filesystem/source/file_pak.cpp
 	common/filesystem/source/file_whres.cpp
 	common/filesystem/source/file_ssi.cpp
+	common/filesystem/source/file_hog.cpp
+	common/filesystem/source/file_mvl.cpp
 	common/filesystem/source/file_directory.cpp
 	common/filesystem/source/resourcefile.cpp
 	common/filesystem/source/files.cpp

src/common/audio/music/music.cpp

@@ -703,6 +703,8 @@ bool S_ChangeMusic(const char* musicname, int order, bool looping, bool force)
 	// opening the music must be done by the game because it's different depending on the game's file system use.
 	FileReader reader = mus_cb.OpenMusic(musicname);
 	if (!reader.isOpen()) return false;
+	auto m = reader.Read();
+	reader.Seek(0, FileReader::SeekSet);
 
 	// shutdown old music
 	S_StopMusic(true);

src/common/filesystem/include/resourcefile.h

@@ -140,6 +140,7 @@ private:
 	int FilterLumps(const std::string& filtername, uint32_t max);
 	bool FindPrefixRange(const char* filter, uint32_t max, uint32_t &start, uint32_t &end);
 	void JunkLeftoverFilters(uint32_t max);
+	void FindCommonFolder(LumpFilterInfo* filter);
 	static FResourceFile *DoOpenResourceFile(const char *filename, FileReader &file, bool containeronly, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 
 public:

src/common/filesystem/source/file_hog.cpp

@@ -0,0 +1,109 @@
+/*
+** file_hog.cpp
+**
+** reads Descent .hog files
+**
+**---------------------------------------------------------------------------
+** Copyright 2023 Christoph Oelckers
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+**
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**---------------------------------------------------------------------------
+**
+**
+*/
+
+
+#include "resourcefile.h"
+#include "fs_swap.h"
+
+namespace FileSys {
+    using namespace byteswap;
+
+
+
+
+static bool OpenHog(FResourceFile* rf, LumpFilterInfo* filter)
+{
+    auto Reader = rf->GetContainerReader();
+    FileReader::Size length = Reader->GetLength();
+
+    std::vector<FResourceEntry> entries;
+    // Hogs store their data as a list of file records, each containing a name, length and the actual data.
+    // To read the directory the entire file must be scanned.
+    while (Reader->Tell() <= length)
+    {
+        char name[13];
+
+        auto r = Reader->Read(&name, 13);
+        if (r < 13) break;
+        name[12] = 0;
+        uint32_t elength = Reader->ReadUInt32();
+
+        FResourceEntry Entry;
+        Entry.Position = Reader->Tell();
+        Entry.Length = elength;
+        Entry.Flags = 0;
+        Entry.CRC32 = 0;
+        Entry.Namespace = ns_global;
+        Entry.ResourceID = -1;
+        Entry.Method = METHOD_STORED;
+        Entry.FileName = rf->NormalizeFileName(name);
+        entries.push_back(Entry);
+        Reader->Seek(elength, FileReader::SeekCur);
+    }
+    auto Entries = rf->AllocateEntries(entries.size());
+    memcpy(Entries, entries.data(), entries.size() * sizeof(Entries[0]));
+    rf->GenerateHash();
+    return true;
+}
+
+
+//==========================================================================
+//
+// File open
+//
+//==========================================================================
+
+FResourceFile* CheckHog(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp)
+{
+    char head[3];
+
+    if (file.GetLength() >= 20)
+    {
+        file.Seek(0, FileReader::SeekSet);
+        file.Read(&head, 3);
+        if (!memcmp(head, "DHF", 3))
+        {
+            auto rf = new FResourceFile(filename, file, sp);
+            if (OpenHog(rf, filter)) return rf;
+            file = rf->Destroy();
+        }
+        file.Seek(0, FileReader::SeekSet);
+    }
+    return nullptr;
+}
+
+
+}

src/common/filesystem/source/file_mvl.cpp

@@ -0,0 +1,98 @@
+/*
+** file_mvl.cpp
+**
+** reads Descent2 .mvl files
+**
+**---------------------------------------------------------------------------
+** Copyright 2023 Christoph Oelckers
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+**
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**---------------------------------------------------------------------------
+**
+**
+*/
+
+
+#include "resourcefile.h"
+#include "fs_swap.h"
+
+namespace FileSys {
+    using namespace byteswap;
+
+
+
+static bool OpenMvl(FResourceFile* rf, LumpFilterInfo* filter)
+{
+    auto Reader = rf->GetContainerReader();
+    auto count = Reader->ReadUInt32();
+    auto Entries = rf->AllocateEntries(count);
+    size_t pos = 8 + (17 * count);   // files start after the directory
+
+    for (uint32_t i = 0; i < count; i++)
+    {
+        char name[13];
+        Reader->Read(&name, 13);
+        name[12] = 0;
+        uint32_t elength = Reader->ReadUInt32();
+
+        Entries[i].Position = pos;
+        Entries[i].Length = elength;
+        Entries[i].ResourceID = -1;
+        Entries[i].FileName = rf->NormalizeFileName(name);
+
+        pos += elength;
+    }
+
+    return true;
+}
+
+
+//==========================================================================
+//
+// File open
+//
+//==========================================================================
+
+FResourceFile* CheckMvl(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp)
+{
+    char head[4];
+
+    if (file.GetLength() >= 20)
+    {
+        file.Seek(0, FileReader::SeekSet);
+        file.Read(&head, 4);
+        if (!memcmp(head, "DMVL", 4))
+        {
+            auto rf = new FResourceFile(filename, file, sp);
+            if (OpenMvl(rf, filter)) return rf;
+            file = rf->Destroy();
+        }
+        file.Seek(0, FileReader::SeekSet);
+    }
+    return nullptr;
+}
+
+
+}

src/common/filesystem/source/file_zip.cpp

@@ -209,55 +209,7 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 			Printf(FSMessageLevel::Error, "%s: Central directory corrupted.", FileName);
 			return false;
 		}
-
-		for (auto& c : name) c = tolower(c);
-
-		auto vv = name.find("__macosx");
-		if (name.find("filter/") == 0)
-			continue; // 'filter' is a reserved name of the file system.
-		if (name.find("__macosx") == 0) 
-			continue; // skip Apple garbage. At this stage only the root folder matters.
-		if (name.find(".bat") != std::string::npos || name.find(".exe") != std::string::npos)
-			continue; // also ignore executables for this.
-		if (!foundprefix)
-		{
-			// check for special names, if one of these gets found this must be treated as a normal zip.
-			bool isspecial = name.find("/") == std::string::npos ||
-				(filter && std::find(filter->reservedFolders.begin(), filter->reservedFolders.end(), name) != filter->reservedFolders.end());
-			if (isspecial) break;
-			name0 = std::string(name, 0, name.rfind("/")+1);
-			name1 = std::string(name, 0, name.find("/") + 1);
-			foundprefix = true;
-		}
-
-		if (name.find(name0) != 0)
-		{
-			if (!name1.empty())
-			{
-				name0 = name1;
-				if (name.find(name0) != 0)
-				{
-					name0 = "";
-				}
-			}
-			if (name0.empty()) 
-				break;
-		}
-		if (!foundspeciallump && filter)
-		{
-			// at least one of the more common definition lumps must be present.
-			for (auto &p : filter->requiredPrefixes)
-			{ 
-				if (name.find(name0 + p) == 0 || name.rfind(p) == size_t(name.length() - p.length()))
-				{
-					foundspeciallump = true;
-					break;
-				}
-			}
-		}
 	}
-	// If it ran through the list without finding anything it should not attempt any path remapping.
-	if (!foundspeciallump) name0 = "";
 
 	dirptr = (char*)directory;
 	AllocateEntries(NumLumps);
@@ -280,13 +232,6 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 			return false;
 		}
 
-		if (name.find("__macosx") == 0 || name.find("__MACOSX") == 0)
-		{
-			skipped++;
-			continue; // Weed out Apple's resource fork garbage right here because it interferes with safe operation.
-		}
-		if (!name0.empty()) name = std::string(name, name0.length());
-
 		// skip Directories
 		if (name.empty() || (name.back() == '/' && LittleLong(zip_fh->UncompressedSize32) == 0))
 		{
@@ -317,9 +262,6 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 			continue;
 		}
 
-		FixPathSeparator(&name.front());
-		for (auto& c : name) c = tolower(c);
-
 		uint32_t UncompressedSize =LittleLong(zip_fh->UncompressedSize32);
 		uint32_t CompressedSize = LittleLong(zip_fh->CompressedSize32);
 		uint64_t LocalHeaderOffset = LittleLong(zip_fh->LocalHeaderOffset32);
@@ -358,7 +300,7 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 		if (Entry->Method != METHOD_STORED) Entry->Flags |= RESFF_COMPRESSED;
 		if (Entry->Method == METHOD_IMPLODE)
 		{
-			// merge the flags into the compression method to tag less data around.
+			// for Implode merge the flags into the compression method to make handling in the file system easier and save one variable.
 			if ((zip_fh->Flags & 6) == 2) Entry->Method = METHOD_IMPLODE_2;
 			else if ((zip_fh->Flags & 6) == 4) Entry->Method = METHOD_IMPLODE_4;
 			else if ((zip_fh->Flags & 6) == 6) Entry->Method = METHOD_IMPLODE_6;

src/common/filesystem/source/resourcefile.cpp

@@ -42,6 +42,7 @@
 #include "unicode.h"
 #include "fs_findfile.h"
 #include "fs_decompress.h"
+#include "wildcards.hpp"
 
 namespace FileSys {
 	
@@ -145,11 +146,13 @@ FResourceFile *CheckPak(const char *filename, FileReader &file, LumpFilterInfo*
 FResourceFile *CheckZip(const char *filename, FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 FResourceFile *Check7Z(const char *filename,  FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 FResourceFile* CheckSSI(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
+FResourceFile* CheckHog(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
+FResourceFile* CheckMvl(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 FResourceFile* CheckWHRes(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 FResourceFile *CheckLump(const char *filename,FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 FResourceFile *CheckDir(const char *filename, bool nosub, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
 
-static CheckFunc funcs[] = { CheckWad, CheckZip, Check7Z, CheckPak, CheckGRP, CheckRFF, CheckSSI, CheckWHRes, CheckLump };
+static CheckFunc funcs[] = { CheckWad, CheckZip, Check7Z, CheckPak, CheckGRP, CheckRFF, CheckSSI, CheckHog, CheckMvl, CheckWHRes, CheckLump };
 
 static int nulPrintf(FSMessageLevel msg, const char* fmt, ...)
 {
@@ -350,10 +353,37 @@ void FResourceFile::GenerateHash()
 void FResourceFile::PostProcessArchive(LumpFilterInfo *filter)
 {
 	// only do this for archive types which contain full file names. All others are assumed to be pre-sorted.
-	if (NumLumps < 2 || !(Entries[0].Flags & RESFF_FULLPATH)) return;
+	if (NumLumps == 0 || !(Entries[0].Flags & RESFF_FULLPATH)) return;
+
+	// First eliminate all unwanted files
+	for (uint32_t i = 0; i < NumLumps; i++)
+	{
+		std::string name = Entries[i].FileName;
+		// remove Apple garbage unconditionally.
+		if (name.find("__macosx") == 0 || name.find("/__macosx") != std::string::npos)
+		{
+			Entries[i].FileName = "";
+			continue;
+		}
+		// Skip executables as well.
+		if (wildcards::match(name, "*.bat") || wildcards::match(name, "*.exe"))
+		{
+			Entries[i].FileName = "";
+			continue;
+		}
+		if (filter) for (auto& pattern : filter->blockednames)
+		{
+			if (wildcards::match(name, pattern))
+			{
+				Entries[i].FileName = "";
+				continue;
+			}
+		}
+	}
 
 	// Entries in archives are sorted alphabetically.
 	qsort(Entries, NumLumps, sizeof(Entries[0]), entrycmp);
+	FindCommonFolder(filter);
 	if (!filter) return;
 
 	// Filter out lumps using the same names as the Autoload.* sections
@@ -377,6 +407,80 @@ void FResourceFile::PostProcessArchive(LumpFilterInfo *filter)
 	JunkLeftoverFilters(max);
 }
 
+//==========================================================================
+//
+// FResourceFile :: FindCommonFolder
+//
+// Checks if all content is in a common folder that can be stripped out.
+//
+//==========================================================================
+
+void FResourceFile::FindCommonFolder(LumpFilterInfo* filter)
+{
+	std::string name0, name1;
+	bool foundspeciallump = false;
+	bool foundprefix = false;
+
+	// try to find a path prefix.
+	for (uint32_t i = 0; i < NumLumps; i++)
+	{
+		if (*Entries[i].FileName == 0) continue;
+		std::string name = Entries[i].FileName;
+
+		// first eliminate files we do not want to have.
+		// Some, like MacOS resource forks and executables are eliminated unconditionally, but the calling code can alsp pass a list of invalid content.
+		if (name.find("filter/") == 0)
+			return; // 'filter' is a reserved name of the file system. If this appears in the root we got no common folder, and 'filter' cannot be it.
+
+		if (!foundprefix)
+		{
+			// check for special names, if one of these gets found this must be treated as a normal zip.
+			bool isspecial = name.find("/") == std::string::npos ||
+				(filter && std::find(filter->reservedFolders.begin(), filter->reservedFolders.end(), name) != filter->reservedFolders.end());
+			if (isspecial) break;
+			name0 = std::string(name, 0, name.rfind("/") + 1);
+			name1 = std::string(name, 0, name.find("/") + 1);
+			foundprefix = true;
+		}
+
+		if (name.find(name0) != 0)
+		{
+			if (!name1.empty())
+			{
+				name0 = name1;
+				if (name.find(name0) != 0)
+				{
+					name0 = "";
+				}
+			}
+			if (name0.empty())
+				break;
+		}
+		if (!foundspeciallump && filter)
+		{
+			// at least one of the more common definition lumps must be present.
+			for (auto& p : filter->requiredPrefixes)
+			{
+				if (name.find(name0 + p) == 0 || name.rfind(p) == size_t(name.length() - p.length()))
+				{
+					foundspeciallump = true;
+					break;
+				}
+			}
+		}
+	}
+	// If it ran through the list without finding anything it should not attempt any path remapping.
+	if (!foundspeciallump || name0.empty()) return;
+
+	size_t pathlen = name0.length();
+	for (uint32_t i = 0; i < NumLumps; i++)
+	{
+		if (Entries[i].FileName[0] == 0) continue;
+		Entries[i].FileName += pathlen;
+
+	}
+}
+
 //==========================================================================
 //
 // FResourceFile :: FilterLumps
@@ -537,13 +641,16 @@ bool FResourceFile::FindPrefixRange(const char* filter, uint32_t maxlump, uint32
 
 int FResourceFile::FindEntry(const char *name)
 {
+	auto norm_fn = tolower_normalize(name);
 	for (unsigned i = 0; i < NumLumps; i++)
 	{
-		if (!stricmp(name, getName(i)))
+		if (!strcmp(norm_fn, getName(i)))
 		{
+			free(norm_fn);
 			return i;
 		}
 	}
+	free(norm_fn);
 	return -1;
 }