- uncoupled directory loader from the rest of the engine.

src/CMakeLists.txt

@@ -1221,6 +1221,7 @@ set( ZDOOM_SOURCES
 	common/filesystem/resourcefile.cpp
 	common/filesystem/files.cpp
 	common/filesystem/files_decompress.cpp
+	common/filesystem/fs_findfile.cpp
 
 )
 

src/common/filesystem/file_directory.cpp

@@ -37,8 +37,11 @@
 #include <sys/stat.h>
 
 #include "resourcefile.h"
-#include "cmdlib.h"
-#include "findfile.h"
+#include "fs_findfile.h"
+
+#ifdef _WIN32
+std::wstring toWide(const char* str);
+#endif
 
 //==========================================================================
 //
@@ -51,7 +54,7 @@ struct FDirectoryLump : public FResourceLump
 	FileReader NewReader() override;
 	int FillCache() override;
 
-	FString mFullPath;
+	std::string mFullPath;
 };
 
 
@@ -66,7 +69,7 @@ class FDirectory : public FResourceFile
 	TArray<FDirectoryLump> Lumps;
 	const bool nosubdir;
 
-	int AddDirectory(const char* dirpath, FileSystemMessageFunc Printf);
+	int AddDirectory(const char* dirpath, LumpFilterInfo* filter, FileSystemMessageFunc Printf);
 	void AddEntry(const char *fullpath, int size);
 
 public:
@@ -86,39 +89,47 @@ public:
 FDirectory::FDirectory(const char * directory, bool nosubdirflag)
 	: FResourceFile(""), nosubdir(nosubdirflag)
 {
-	FString dirname;
-
-	#ifdef _WIN32
-		directory = _fullpath(NULL, directory, _MAX_PATH);
-	#else
-		// Todo for Linux: Resolve the path before using it
-	#endif
-	dirname = directory;
-#ifdef _WIN32
-	free((void *)directory);
-#endif
-	dirname.Substitute("\\", "/");
-	if (dirname[dirname.Len()-1] != '/') dirname += '/';
-	FileName = dirname;
+	FileName = FS_FullPath(directory);
+	if (FileName[FileName.length()-1] != '/') FileName += '/';
 }
 
-
 //==========================================================================
 //
 // Windows version
 //
 //==========================================================================
 
-int FDirectory::AddDirectory(const char *dirpath, FileSystemMessageFunc Printf)
+static bool FS_GetFileInfo(const char* pathname, size_t* size)
+{
+#ifndef _WIN32
+	struct stat info;
+	bool res = stat(pathname, &info) == 0;
+#else
+	// Windows must use the wide version of stat to preserve non-ASCII paths.
+	struct _stat64 info;
+	bool res = _wstat64(toWide(pathname).c_str(), &info) == 0;
+#endif
+	if (!res || (info.st_mode & S_IFDIR)) return false;
+	if (size) *size = (size_t)info.st_size;
+	return res;
+}
+
+//==========================================================================
+//
+// Windows version
+//
+//==========================================================================
+
+int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 {
 	void * handle;
 	int count = 0;
 
-	FString dirmatch = dirpath;
-	findstate_t find;
+	std::string dirmatch = dirpath;
 	dirmatch += '*';
+	fs_findstate_t find;
 
-	handle = I_FindFirst(dirmatch.GetChars(), &find);
+	handle = FS_FindFirst(dirmatch.c_str(), &find);
 	if (handle == ((void *)(-1)))
 	{
 		Printf(FSMessageLevel::Error, "Could not scan '%s': %s\n", dirpath, strerror(errno));
@@ -127,15 +138,15 @@ int FDirectory::AddDirectory(const char *dirpath, FileSystemMessageFunc Printf)
 	{
 		do
 		{
-			// I_FindName only returns the file's name and not its full path
-			auto attr = I_FindAttr(&find);
+			// FS_FindName only returns the file's name and not its full path
+			auto attr = FS_FindAttr(&find);
 			if (attr & FA_HIDDEN)
 			{
-				// Skip hidden files and directories. (Prevents SVN bookkeeping
+				// Skip hidden files and directories. (Prevents SVN/Git bookkeeping
 				// info from being included.)
 				continue;
 			}
-			FString fi = I_FindName(&find);
+			const char* fi = FS_FindName(&find);
 			if (attr &  FA_DIREC)
 			{
 				if (nosubdir || (fi[0] == '.' &&
@@ -145,9 +156,10 @@ int FDirectory::AddDirectory(const char *dirpath, FileSystemMessageFunc Printf)
 					// Do not record . and .. directories.
 					continue;
 				}
-				FString newdir = dirpath;
-				newdir << fi << '/';
-				count += AddDirectory(newdir, Printf);
+				std::string newdir = dirpath;
+				newdir += fi;
+				newdir += '/';
+				count += AddDirectory(newdir.c_str(), filter, Printf);
 			}
 			else
 			{
@@ -157,31 +169,25 @@ int FDirectory::AddDirectory(const char *dirpath, FileSystemMessageFunc Printf)
 					continue;
 				}
 				size_t size = 0;
-				FString fn = FString(dirpath) + fi;
+				std::string fn = dirpath;
+				fn += fi;
 
-				// The next one is courtesy of EDuke32. :(
-				// Putting cache files in the application directory is very bad style.
-				// Unfortunately, having a garbage file named "texture" present will cause serious problems down the line.
-				if (!stricmp(fi, "textures"))
+				if (filter->filenamecheck == nullptr || filter->filenamecheck(fi, fn.c_str()))
 				{
-					FILE* f = fopen(fn, "rb");
-					if (f)
+					if (FS_GetFileInfo(fn.c_str(), &size))
 					{
-						char check[3]{};
-						fread(check, 1, 3, f);
-						if (!memcmp(check, "LZ4", 3)) continue;
+						if (size > 0x7fffffff)
+						{
+							Printf(FSMessageLevel::Warning, "%s is larger than 2GB and will be ignored\n", fn.c_str());
+						}
+						AddEntry(fn.c_str(), (int)size);
+						count++;
 					}
 				}
-
-				if (GetFileInfo(fn, &size, nullptr))
-				{
-					AddEntry(fn, (int)size);
-					count++;
-				}
 			}
 
-		} while (I_FindNext (handle, &find) == 0);
-		I_FindClose (handle);
+		} while (FS_FindNext (handle, &find) == 0);
+		FS_FindClose (handle);
 	}
 	return count;
 }
@@ -194,7 +200,7 @@ int FDirectory::AddDirectory(const char *dirpath, FileSystemMessageFunc Printf)
 
 bool FDirectory::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
 {
-	NumLumps = AddDirectory(FileName.c_str(), Printf);
+	NumLumps = AddDirectory(FileName.c_str(), filter, Printf);
 	PostProcessArchive(&Lumps[0], sizeof(FDirectoryLump), filter);
 	return true;
 }
@@ -234,7 +240,7 @@ void FDirectory::AddEntry(const char *fullpath, int size)
 FileReader FDirectoryLump::NewReader()
 {
 	FileReader fr;
-	fr.OpenFile(mFullPath);
+	fr.OpenFile(mFullPath.c_str());
 	return fr;
 }
 
@@ -248,7 +254,7 @@ int FDirectoryLump::FillCache()
 {
 	FileReader fr;
 	Cache = new char[LumpSize];
-	if (!fr.OpenFile(mFullPath))
+	if (!fr.OpenFile(mFullPath.c_str()))
 	{
 		throw FileSystemException("unable to open file");
 	}

src/common/filesystem/files.cpp

@@ -33,18 +33,21 @@
 **
 */
 
+#include <string>
 #include "files.h"
-#include "utf8.h"
 #include "stb_sprintf.h"
 
+#ifdef _WIN32
+std::wstring toWide(const char* str);
+#endif
 
 FILE *myfopen(const char *filename, const char *flags)
 {
 #ifndef _WIN32
 	return fopen(filename, flags);
 #else
-	auto widename = WideString(filename);
-	auto wideflags = WideString(flags);
+	auto widename = toWide(filename);
+	auto wideflags = toWide(flags);
 	return _wfopen(widename.c_str(), wideflags.c_str());
 #endif
 }

src/common/filesystem/fs_findfile.cpp

@@ -0,0 +1,232 @@
+/*
+** findfile.cpp
+** Wrapper around the native directory scanning APIs
+**
+**---------------------------------------------------------------------------
+** Copyright 1998-2016 Randy Heit
+** Copyright 2005-2020 Christoph Oelckers
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+**
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**---------------------------------------------------------------------------
+**
+*/
+
+#include "fs_findfile.h"
+
+#ifndef _WIN32
+
+#include <unistd.h>
+#include <fnmatch.h>
+#include <sys/stat.h>
+
+static const char *pattern;
+
+static int matchfile(const struct dirent *ent)
+{
+	return fnmatch(pattern, ent->d_name, FNM_NOESCAPE) == 0;
+}
+
+void *FS_FindFirst(const char *const filespec, fs_findstate_t *const fileinfo)
+{
+	const char* dir;
+
+	const char *const slash = strrchr(filespec, '/');
+
+	if (slash)
+	{
+		pattern = slash + 1;
+		fileinfo->path = std::string(filespec, slash - filespec + 1);
+		dir = fileinfo->path.c_str();
+	}
+	else
+	{
+		pattern = filespec;
+		dir = ".";
+	}
+
+	fileinfo->current = 0;
+	fileinfo->count = scandir(dir, &fileinfo->namelist, matchfile, alphasort);
+
+	if (fileinfo->count > 0)
+	{
+		return fileinfo;
+	}
+
+	return (void *)-1;
+}
+
+int FS_FindNext(void *const handle, fs_findstate_t *const fileinfo)
+{
+	fs_findstate_t *const state = static_cast<fs_findstate_t *>(handle);
+
+	if (state->current < fileinfo->count)
+	{
+		return ++state->current < fileinfo->count ? 0 : -1;
+	}
+
+	return -1;
+}
+
+int FS_FindClose(void *const handle)
+{
+	fs_findstate_t *const state = static_cast<fs_findstate_t *>(handle);
+
+	if (handle != (void *)-1 && state->count > 0)
+	{
+		for (int i = 0; i < state->count; ++i)
+		{
+			free(state->namelist[i]);
+		}
+
+		free(state->namelist);
+		state->namelist = nullptr;
+		state->count = 0;
+	}
+
+	return 0;
+}
+
+static bool DirEntryExists(const char* pathname, bool* isdir)
+{
+	if (isdir) *isdir = false;
+	struct stat info;
+	bool res = stat(pathname, &info) == 0;
+	if (isdir) *isdir = !!(info.st_mode & S_IFDIR);
+	return res;
+}
+
+int FS_FindAttr(fs_findstate_t *const fileinfo)
+{
+	dirent *const ent = fileinfo->namelist[fileinfo->current];
+	const std::string path = fileinfo->path + ent->d_name;
+	bool isdir;
+
+	if (DirEntryExists(path.c_str(), &isdir))
+	{
+		return isdir ? FA_DIREC : 0;
+	}
+
+	return 0;
+}
+
+std::string FS_FullPath(const char* directory)
+{
+	// todo
+	return directory
+}
+
+#else
+
+#include <windows.h>
+#include <direct.h>
+
+std::wstring toWide(const char* str)
+{
+	int len = (int)strlen(str);
+	int size_needed = MultiByteToWideChar(CP_UTF8, 0, str, len, nullptr, 0);
+	std::wstring wide;
+	wide.resize(size_needed);
+	MultiByteToWideChar(CP_UTF8, 0, str, len, &wide.front(), size_needed);
+	return wide;
+}
+
+static std::string toUtf8(const wchar_t* str)
+{
+	auto len = wcslen(str);
+	int size_needed = WideCharToMultiByte(CP_UTF8, 0, str, (int)len, nullptr, 0, nullptr, nullptr);
+	std::string utf8;
+	utf8.resize(size_needed);
+	WideCharToMultiByte(CP_UTF8, 0, str, (int)len, &utf8.front(), size_needed, nullptr, nullptr);
+	return utf8;
+}
+
+//==========================================================================
+//
+// FS_FindFirst
+//
+// Start a pattern matching sequence.
+//
+//==========================================================================
+
+
+void *FS_FindFirst(const char *filespec, fs_findstate_t *fileinfo)
+{
+	static_assert(sizeof(WIN32_FIND_DATAW) == sizeof(fileinfo->FindData), "FindData size mismatch");
+	fileinfo->UTF8Name = "";
+	return FindFirstFileW(toWide(filespec).c_str(), (LPWIN32_FIND_DATAW)&fileinfo->FindData);
+}
+
+//==========================================================================
+//
+// FS_FindNext
+//
+// Return the next file in a pattern matching sequence.
+//
+//==========================================================================
+
+int FS_FindNext(void *handle, fs_findstate_t *fileinfo)
+{
+	fileinfo->UTF8Name = "";
+	return FindNextFileW((HANDLE)handle, (LPWIN32_FIND_DATAW)&fileinfo->FindData) == 0;
+}
+
+//==========================================================================
+//
+// FS_FindClose
+//
+// Finish a pattern matching sequence.
+//
+//==========================================================================
+
+int FS_FindClose(void *handle)
+{
+	return FindClose((HANDLE)handle);
+}
+
+//==========================================================================
+//
+// FS_FindName
+//
+// Returns the name for an entry
+//
+//==========================================================================
+
+const char *FS_FindName(fs_findstate_t *fileinfo)
+{
+	if (fileinfo->UTF8Name.empty())
+	{
+		fileinfo->UTF8Name = toUtf8(fileinfo->FindData.Name);
+	}
+	return fileinfo->UTF8Name.c_str();
+}
+
+std::string FS_FullPath(const char* directory)
+{
+	auto wdirectory = _wfullpath(nullptr, toWide(directory).c_str(), _MAX_PATH);
+	std::string sdirectory = toUtf8(wdirectory);
+	free((void*)wdirectory);
+	for (auto& c : sdirectory) if (c == '\\') c = '/';
+	return sdirectory;
+}
+#endif

src/common/filesystem/fs_findfile.h

@@ -0,0 +1,98 @@
+#pragma once
+// Directory searching routines
+
+#include <stdint.h>
+#include <string>
+
+enum
+{
+	ZPATH_MAX = 260
+};
+
+#ifndef _WIN32
+
+#include <dirent.h>
+
+struct fs_findstate_t
+{
+private:
+	std::string path;
+    struct dirent **namelist;
+    int current;
+    int count;
+
+	friend void *FS_FindFirst(const char *filespec, fs_findstate_t *fileinfo);
+	friend int FS_FindNext(void *handle, fs_findstate_t *fileinfo);
+	friend const char *FS_FindName(fs_findstate_t *fileinfo);
+	friend int FS_FindAttr(fs_findstate_t *fileinfo);
+	friend int FS_FindClose(void *handle);
+};
+
+int FS_FindAttr (fs_findstate_t *fileinfo); 
+
+inline const char *FS_FindName(fs_findstate_t *fileinfo)
+{
+	return (fileinfo->namelist[fileinfo->current]->d_name);
+}
+
+enum
+{
+	FA_RDONLY	= 1,
+	FA_HIDDEN	= 2,
+	FA_SYSTEM	= 4,
+	FA_DIREC	= 8,
+	FA_ARCH		= 16,
+};
+
+
+#else
+
+
+struct fs_findstate_t
+{
+private:
+	struct FileTime
+	{
+		uint32_t lo, hi;
+	};
+	// Mirror WIN32_FIND_DATAW in <winbase.h>. We cannot pull in the Windows header here as it would pollute all consumers' symbol space.
+	struct WinData
+	{
+		uint32_t Attribs;
+		FileTime Times[3];
+		uint32_t Size[2];
+		uint32_t Reserved[2];
+		wchar_t Name[ZPATH_MAX];
+		wchar_t AltName[14];
+	};
+	WinData FindData;
+	std::string UTF8Name;
+
+	friend void *FS_FindFirst(const char *filespec, fs_findstate_t *fileinfo);
+	friend int FS_FindNext(void *handle, fs_findstate_t *fileinfo);
+	friend const char *FS_FindName(fs_findstate_t *fileinfo);
+	friend int FS_FindAttr(fs_findstate_t *fileinfo);
+};
+
+
+const char *FS_FindName(fs_findstate_t *fileinfo);
+inline int FS_FindAttr(fs_findstate_t *fileinfo)
+{
+	return fileinfo->FindData.Attribs;
+}
+
+enum
+{
+	FA_RDONLY	= 1,
+	FA_HIDDEN	= 2,
+	FA_SYSTEM	= 4,
+	FA_DIREC	= 16,
+	FA_ARCH		= 32,
+};
+
+#endif
+
+void *FS_FindFirst (const char *filespec, fs_findstate_t *fileinfo);
+int FS_FindNext (void *handle, fs_findstate_t *fileinfo);
+int FS_FindClose (void *handle);
+std::string FS_FullPath(const char* directory);

src/common/filesystem/resourcefile.h

@@ -21,6 +21,8 @@ struct LumpFilterInfo
 	std::vector<std::string> reservedFolders;
 	std::vector<std::string> requiredPrefixes;
 	std::vector<std::string> embeddings;
+	std::vector<std::string> blockednames;			// File names that will never be accepted (e.g. dehacked.exe for Doom)
+	std::function<bool(const char*, const char*)> filenamecheck;	// for scanning directories, this allows to eliminate unwanted content.
 	std::function<void()> postprocessFunc;
 };
 

src/common/filesystem/w_zip.h

@@ -1,7 +1,11 @@
 #ifndef __W_ZIP
 #define __W_ZIP
 
-#include "basics.h"
+#if defined(__GNUC__)
+#define FORCE_PACKED __attribute__((__packed__))
+#else
+#define FORCE_PACKED
+#endif
 
 #pragma pack(1)
 // FZipCentralInfo
@@ -85,6 +89,15 @@ struct FZipLocalFileHeader
 
 #pragma pack()
 
+#ifndef MAKE_ID
+#ifndef __BIG_ENDIAN__
+#define MAKE_ID(a,b,c,d)	((uint32_t)((a)|((b)<<8)|((c)<<16)|((d)<<24)))
+#else
+#define MAKE_ID(a,b,c,d)	((uint32_t)((d)|((c)<<8)|((b)<<16)|((a)<<24)))
+#endif
+#endif
+
+
 #define ZIP_LOCALFILE	MAKE_ID('P','K',3,4)
 #define ZIP_CENTRALFILE	MAKE_ID('P','K',1,2)
 #define ZIP_ENDOFDIR	MAKE_ID('P','K',5,6)

src/d_main.cpp

@@ -3034,6 +3034,26 @@ static FILE* D_GetHashFile()
 	return hashfile;
 }
 
+// checks if a file within a directory is allowed to be added to the file system.
+static bool FileNameCheck(const char* base, const char* path)
+{
+	// This one is courtesy of EDuke32. :(
+	// Putting cache files in the application directory is very bad style.
+	// Unfortunately, having a garbage file named "textures" present will cause serious problems down the line.
+	if (!strnicmp(base, "textures", 8))
+	{
+		// do not use fopen. The path may contain non-ASCII characters.
+		FileReader f;
+		if (f.OpenFile(path))
+		{
+			char check[3]{};
+			f.Read(check, 3);
+			if (!memcmp(check, "LZ4", 3)) return false;
+		}
+	}
+	return true;
+}
+
 static int FileSystemPrintf(FSMessageLevel level, const char* fmt, ...)
 {
 	va_list arg;