From 36930d44bde19e1d15e769a905978161304b907c Mon Sep 17 00:00:00 2001 From: Christoph Oelckers Date: Sun, 17 Dec 2023 12:48:03 +0100 Subject: [PATCH] filesystem update from GZDoom. --- source/CMakeLists.txt | 4 + source/common/audio/music/i_music.cpp | 8 +- source/common/audio/music/i_soundfont.cpp | 2 +- source/common/audio/music/music.cpp | 2 + source/common/console/c_cvars.cpp | 57 +- source/common/console/c_cvars.h | 9 +- source/common/cutscenes/movieplayer.cpp | 12 +- source/common/engine/sc_man.cpp | 8 +- source/common/engine/serializer.cpp | 1 - source/common/engine/stringtable.cpp | 6 +- .../common/filesystem/include/fs_decompress.h | 63 + source/common/filesystem/include/fs_files.h | 90 +- .../common/filesystem/include/fs_filesystem.h | 53 +- .../common/filesystem/include/fs_findfile.h | 1 - .../common/filesystem/include/resourcefile.h | 182 +- .../filesystem/{source => include}/w_zip.h | 4 +- .../common/filesystem/source/ancientzip.cpp | 17 +- source/common/filesystem/source/ancientzip.h | 16 +- source/common/filesystem/source/critsec.cpp | 150 ++ source/common/filesystem/source/critsec.h | 39 + source/common/filesystem/source/file_7z.cpp | 102 +- .../filesystem/source/file_directory.cpp | 101 +- source/common/filesystem/source/file_grp.cpp | 59 +- source/common/filesystem/source/file_hog.cpp | 109 + source/common/filesystem/source/file_lump.cpp | 48 +- source/common/filesystem/source/file_mvl.cpp | 98 + source/common/filesystem/source/file_pak.cpp | 66 +- source/common/filesystem/source/file_rff.cpp | 145 +- source/common/filesystem/source/file_ssi.cpp | 76 +- source/common/filesystem/source/file_wad.cpp | 209 +- .../common/filesystem/source/file_whres.cpp | 77 +- source/common/filesystem/source/file_zip.cpp | 449 +--- source/common/filesystem/source/file_zip.h | 51 - source/common/filesystem/source/files.cpp | 62 +- .../filesystem/source/files_decompress.cpp | 129 +- .../common/filesystem/source/files_internal.h | 23 +- .../common/filesystem/source/filesystem.cpp | 177 +- .../filesystem/source/fs_stringpool.cpp | 5 +- .../common/filesystem/source/fs_stringpool.h | 2 +- .../common/filesystem/source/resourcefile.cpp | 643 +++--- .../filesystem/source/resourcefile_internal.h | 38 - source/common/filesystem/source/unicode.cpp | 20 +- source/common/filesystem/source/unicode.h | 2 +- source/common/filesystem/source/wildcards.hpp | 1833 +++++++++++++++++ source/common/fonts/singlelumpfont.cpp | 4 +- source/common/menu/savegamemanager.cpp | 2 +- source/common/models/model.cpp | 2 +- source/common/models/models_iqm.cpp | 2 +- source/common/models/models_md2.cpp | 4 +- source/common/models/models_md3.cpp | 2 +- source/common/models/models_ue1.cpp | 4 +- source/common/models/voxels.cpp | 4 +- .../common/rendering/gl/gl_shaderprogram.cpp | 1 - source/common/rendering/gles/gles_shader.cpp | 3 - .../rendering/gles/gles_shaderprogram.cpp | 1 - .../rendering/vulkan/shaders/vk_ppshader.cpp | 1 - .../rendering/vulkan/shaders/vk_shader.cpp | 1 - source/common/textures/formats/anmtexture.cpp | 4 +- .../textures/formats/automaptexture.cpp | 2 +- .../common/textures/formats/imgztexture.cpp | 2 +- .../common/textures/formats/patchtexture.cpp | 4 +- source/common/textures/formats/qoitexture.cpp | 6 +- .../textures/formats/rawpagetexture.cpp | 6 +- .../textures/formats/startuptexture.cpp | 12 +- .../common/textures/formats/webptexture.cpp | 2 +- .../textures/multipatchtexturebuilder.cpp | 4 +- source/common/textures/texturemanager.cpp | 6 +- source/common/utility/cmdlib.cpp | 2 +- source/common/utility/palette.cpp | 12 +- source/common/utility/writezip.cpp | 239 +++ source/core/initfs.cpp | 7 +- source/core/savegamehelp.cpp | 6 +- source/games/blood/src/blood.cpp | 8 +- source/games/blood/src/sound.cpp | 2 +- source/games/exhumed/src/sound.cpp | 2 +- source/games/sw/src/scrip2.cpp | 2 +- 76 files changed, 3693 insertions(+), 1914 deletions(-) create mode 100644 source/common/filesystem/include/fs_decompress.h rename source/common/filesystem/{source => include}/w_zip.h (99%) create mode 100644 source/common/filesystem/source/critsec.cpp create mode 100644 source/common/filesystem/source/critsec.h create mode 100644 source/common/filesystem/source/file_hog.cpp create mode 100644 source/common/filesystem/source/file_mvl.cpp delete mode 100644 source/common/filesystem/source/file_zip.h delete mode 100644 source/common/filesystem/source/resourcefile_internal.h create mode 100644 source/common/filesystem/source/wildcards.hpp create mode 100644 source/common/utility/writezip.cpp diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index f4a8a656e..f342701b7 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1122,6 +1122,7 @@ set (PCH_SOURCES common/utility/s_playlist.cpp common/utility/name.cpp common/utility/r_memory.cpp + common/utility/writezip.cpp common/thirdparty/base64.cpp common/thirdparty/md5.cpp common/thirdparty/superfasthash.cpp @@ -1279,6 +1280,8 @@ set( GAME_SOURCES common/filesystem/source/file_pak.cpp common/filesystem/source/file_whres.cpp common/filesystem/source/file_ssi.cpp + common/filesystem/source/file_hog.cpp + common/filesystem/source/file_mvl.cpp common/filesystem/source/file_directory.cpp common/filesystem/source/resourcefile.cpp common/filesystem/source/files.cpp @@ -1286,6 +1289,7 @@ set( GAME_SOURCES common/filesystem/source/fs_findfile.cpp common/filesystem/source/fs_stringpool.cpp common/filesystem/source/unicode.cpp + common/filesystem/source/critsec.cpp ) diff --git a/source/common/audio/music/i_music.cpp b/source/common/audio/music/i_music.cpp index 1f17cf884..d190537d4 100644 --- a/source/common/audio/music/i_music.cpp +++ b/source/common/audio/music/i_music.cpp @@ -182,8 +182,8 @@ static void SetupGenMidi() } auto genmidi = fileSystem.ReadFile(lump); - if (genmidi.GetSize() < 8 + 175 * 36 || memcmp(genmidi.GetMem(), "#OPL_II#", 8)) return; - ZMusic_SetGenMidi(genmidi.GetBytes() + 8); + if (genmidi.size() < 8 + 175 * 36 || memcmp(genmidi.data(), "#OPL_II#", 8)) return; + ZMusic_SetGenMidi(genmidi.bytes() + 8); } static void SetupWgOpn() @@ -194,7 +194,7 @@ static void SetupWgOpn() return; } auto data = fileSystem.ReadFile(lump); - ZMusic_SetWgOpn(data.GetMem(), (uint32_t)data.GetSize()); + ZMusic_SetWgOpn(data.data(), (uint32_t)data.size()); } static void SetupDMXGUS() @@ -206,7 +206,7 @@ static void SetupDMXGUS() return; } auto data = fileSystem.ReadFile(lump); - ZMusic_SetDmxGus(data.GetMem(), (uint32_t)data.GetSize()); + ZMusic_SetDmxGus(data.data(), (uint32_t)data.size()); } #endif diff --git a/source/common/audio/music/i_soundfont.cpp b/source/common/audio/music/i_soundfont.cpp index e4a6bc39f..0f3c0020b 100644 --- a/source/common/audio/music/i_soundfont.cpp +++ b/source/common/audio/music/i_soundfont.cpp @@ -218,7 +218,7 @@ FileReader FZipPatReader::OpenFile(const char *name) auto lump = resf->FindEntry(name); if (lump >= 0) { - return resf->GetEntryReader(lump); + return resf->GetEntryReader(lump, FileSys::READER_NEW, FileSys::READERFLAG_SEEKABLE); } } fr.OpenFile(name); diff --git a/source/common/audio/music/music.cpp b/source/common/audio/music/music.cpp index 51797220e..43e05cb5c 100644 --- a/source/common/audio/music/music.cpp +++ b/source/common/audio/music/music.cpp @@ -703,6 +703,8 @@ bool S_ChangeMusic(const char* musicname, int order, bool looping, bool force) // opening the music must be done by the game because it's different depending on the game's file system use. FileReader reader = mus_cb.OpenMusic(musicname); if (!reader.isOpen()) return false; + auto m = reader.Read(); + reader.Seek(0, FileReader::SeekSet); // shutdown old music S_StopMusic(true); diff --git a/source/common/console/c_cvars.cpp b/source/common/console/c_cvars.cpp index e8d94acea..1cbe6772d 100644 --- a/source/common/console/c_cvars.cpp +++ b/source/common/console/c_cvars.cpp @@ -44,6 +44,7 @@ #include "printf.h" #include "palutil.h" #include "i_interface.h" +#include "gstrings.h" #include "dobject.h" #include "dobjtype.h" @@ -1695,16 +1696,37 @@ CCMD (toggle) } } -void FBaseCVar::ListVars (const char *filter, bool plain) +void FBaseCVar::ListVars (const char *filter, int listtype) { int count = 0; + bool plain = listtype == LCT_Plain; + bool includedesc = listtype == LCT_FullSearch; + decltype(cvarMap)::Iterator it(cvarMap); decltype(cvarMap)::Pair *pair; while (it.NextPair(pair)) { auto var = pair->Value; - if (CheckWildcards (filter, var->GetName())) + + bool ismatch; + + if (filter && includedesc) + { + // search always allow partial matches + // also allow matching to cvar name, localised description, and description language-id + + FString SearchString = FString("*") + filter + "*"; + ismatch = CheckWildcards (SearchString.GetChars(), var->GetName()) || + CheckWildcards (SearchString.GetChars(), var->GetDescription().GetChars()) || + CheckWildcards (SearchString.GetChars(), GStrings.localize(var->GetDescription().GetChars())); + } + else + { + ismatch = CheckWildcards (filter, var->GetName()); + } + + if (ismatch) { uint32_t flags = var->GetFlags(); if (plain) @@ -1718,7 +1740,8 @@ void FBaseCVar::ListVars (const char *filter, bool plain) else { ++count; - Printf ("%c%c%c%c%c %s = %s\n", + + Printf ("%c%c%c%c%c %s = %s", flags & CVAR_ARCHIVE ? 'A' : ' ', flags & CVAR_USERINFO ? 'U' : flags & CVAR_SERVERINFO ? 'S' : @@ -1730,6 +1753,16 @@ void FBaseCVar::ListVars (const char *filter, bool plain) flags & CVAR_IGNORE ? 'X' : ' ', var->GetName(), var->GetHumanString()); + + if (includedesc) + if (var->GetDescription().Len()) + Printf(" // \"%s\"\n", GStrings.localize(var->GetDescription().GetChars())); + else + Printf("\n"); + else + Printf("\n"); + + } } } @@ -1740,17 +1773,29 @@ CCMD (cvarlist) { if (argv.argc() == 1) { - FBaseCVar::ListVars (NULL, false); + FBaseCVar::ListVars (NULL, LCT_Default); } else { - FBaseCVar::ListVars (argv[1], false); + FBaseCVar::ListVars (argv[1], LCT_Default); } } CCMD (cvarlistplain) { - FBaseCVar::ListVars (NULL, true); + FBaseCVar::ListVars (NULL, LCT_Plain); +} + +CCMD (cvarsearch) +{ + if (argv.argc() == 1) + { + FBaseCVar::ListVars (NULL, LCT_FullSearch); + } + else + { + FBaseCVar::ListVars (argv[1], LCT_FullSearch); + } } CCMD (archivecvar) diff --git a/source/common/console/c_cvars.h b/source/common/console/c_cvars.h index e9fc87420..47337523f 100644 --- a/source/common/console/c_cvars.h +++ b/source/common/console/c_cvars.h @@ -89,6 +89,13 @@ enum ECVarType CVAR_Dummy, // Unknown }; +enum ListCCMDType +{ + LCT_Default, + LCT_Plain, + LCT_FullSearch, +}; + class FIntCVarRef; union UCVarValue @@ -201,7 +208,7 @@ public: static void MarkZSCallbacks (); static void ResetColors (); // recalc color cvars' indices after screen change - static void ListVars (const char *filter, bool plain); + static void ListVars (const char *filter, int listtype); const FString &GetDescription() const { return Description; }; const FString& GetToggleMessage(int which) { return ToggleMessages[which]; } diff --git a/source/common/cutscenes/movieplayer.cpp b/source/common/cutscenes/movieplayer.cpp index 712467899..d9b02541e 100644 --- a/source/common/cutscenes/movieplayer.cpp +++ b/source/common/cutscenes/movieplayer.cpp @@ -153,7 +153,7 @@ class AnmPlayer : public MoviePlayer { // This doesn't need its own class type anim_t anim; - FileSys::ResourceData buffer; + FileSys::FileData buffer; int numframes = 0; int curframe = 1; int frametime = 0; @@ -515,7 +515,7 @@ public: } else if (soundtrack >= 0) { - FileReader reader = fileSystem.OpenFileReader(soundtrack); + FileReader reader = fileSystem.ReopenFileReader(soundtrack); if (reader.isOpen()) { MusicStream = ZMusic_OpenSong(GetMusicReader(reader), MDEV_DEFAULT, nullptr); @@ -794,7 +794,7 @@ public: if (sound == INVALID_SOUND) soundEngine->StopAllChannels(); else - soundEngine->StartSound(SOURCE_None, nullptr, nullptr, CHAN_AUTO, nostopsound ? CHANF_UI | CHANF_FORCE : CHANF_FORCE, sound, 1.f, ATTN_NONE); + soundEngine->StartSound(SOURCE_None, nullptr, nullptr, CHAN_AUTO, nostopsound ? CHANF_UI : CHANF_NONE, sound, 1.f, ATTN_NONE); } } } @@ -838,10 +838,10 @@ MoviePlayer* OpenMovie(const char* filename, TArray& ans, const int* framet { auto fn = StripExtension(filename); DefaultExtension(fn, ".ivf"); - fr = fileSystem.OpenFileReader(fn.GetChars()); + fr = fileSystem.ReopenFileReader(fn.GetChars()); } - if (!fr.isOpen()) fr = fileSystem.OpenFileReader(filename); + if (!fr.isOpen()) fr = fileSystem.ReopenFileReader(filename); if (!fr.isOpen()) { size_t nLen = strlen(filename); @@ -849,7 +849,7 @@ MoviePlayer* OpenMovie(const char* filename, TArray& ans, const int* framet if (nLen >= 3 && isalpha(filename[0]) && filename[1] == ':' && filename[2] == '/') { filename += 3; - fr = fileSystem.OpenFileReader(filename); + fr = fileSystem.ReopenFileReader(filename); } if (!fr.isOpen()) { diff --git a/source/common/engine/sc_man.cpp b/source/common/engine/sc_man.cpp index e8423178d..9849c2e21 100644 --- a/source/common/engine/sc_man.cpp +++ b/source/common/engine/sc_man.cpp @@ -200,10 +200,10 @@ void FScanner :: OpenLumpNum (int lump) { Close (); { - auto mem = fileSystem.OpenFileReader(lump); - auto buff = ScriptBuffer.LockNewBuffer(mem.GetLength()); - mem.Read(buff, mem.GetLength()); - buff[mem.GetLength()] = 0; + auto len = fileSystem.FileLength(lump); + auto buff = ScriptBuffer.LockNewBuffer(len); + fileSystem.ReadFile(lump, buff); + buff[len] = 0; ScriptBuffer.UnlockBuffer(); } ScriptName = fileSystem.GetFileFullPath(lump).c_str(); diff --git a/source/common/engine/serializer.cpp b/source/common/engine/serializer.cpp index ae0ad5ffc..261aa464c 100644 --- a/source/common/engine/serializer.cpp +++ b/source/common/engine/serializer.cpp @@ -753,7 +753,6 @@ FCompressedBuffer FSerializer::GetCompressedOutput() EndObject(); buff.filename = nullptr; buff.mSize = (unsigned)w->mOutString.GetSize(); - buff.mZipFlags = 0; buff.mCRC32 = crc32(0, (const Bytef*)w->mOutString.GetString(), buff.mSize); uint8_t *compressbuf = new uint8_t[buff.mSize+1]; diff --git a/source/common/engine/stringtable.cpp b/source/common/engine/stringtable.cpp index d507b281c..6bfbe1280 100644 --- a/source/common/engine/stringtable.cpp +++ b/source/common/engine/stringtable.cpp @@ -63,8 +63,8 @@ void FStringTable::LoadStrings (const char *language) { auto lumpdata = fileSystem.ReadFile(lump); - if (!ParseLanguageCSV(lump, lumpdata.GetString(), lumpdata.GetSize())) - LoadLanguage (lump, lumpdata.GetString(), lumpdata.GetSize()); + if (!ParseLanguageCSV(lump, lumpdata.string(), lumpdata.size())) + LoadLanguage (lump, lumpdata.string(), lumpdata.size()); } UpdateLanguage(language); allMacros.Clear(); @@ -160,7 +160,7 @@ TArray> FStringTable::parseCSV(const char* buffer, size_t size) bool FStringTable::readMacros(int lumpnum) { auto lumpdata = fileSystem.ReadFile(lumpnum); - auto data = parseCSV(lumpdata.GetString(), lumpdata.GetSize()); + auto data = parseCSV(lumpdata.string(), lumpdata.size()); for (unsigned i = 1; i < data.Size(); i++) { diff --git a/source/common/filesystem/include/fs_decompress.h b/source/common/filesystem/include/fs_decompress.h new file mode 100644 index 000000000..0ea062ddb --- /dev/null +++ b/source/common/filesystem/include/fs_decompress.h @@ -0,0 +1,63 @@ +#pragma once +#include "fs_files.h" + +namespace FileSys { + +// Zip compression methods, extended by some internal types to be passed to OpenDecompressor +enum ECompressionMethod +{ + METHOD_STORED = 0, + METHOD_SHRINK = 1, + METHOD_IMPLODE = 6, + METHOD_DEFLATE = 8, + METHOD_BZIP2 = 12, + METHOD_LZMA = 14, + METHOD_XZ = 95, + METHOD_PPMD = 98, + METHOD_LZSS = 1337, // not used in Zips - this is for Console Doom compression + METHOD_ZLIB = 1338, // Zlib stream with header, used by compressed nodes. + METHOD_RFFCRYPT = 1339, // not actual compression but can be put in here to make handling easier. + METHOD_IMPLODE_MIN = 1000, // having discrete types for these avoids keeping around the GPFlags word in Zips. + METHOD_IMPLODE_0 = 1000, + METHOD_IMPLODE_2 = 1002, + METHOD_IMPLODE_4 = 1004, + METHOD_IMPLODE_6 = 1006, + METHOD_IMPLODE_MAX = 1006, + METHOD_INVALID = 0x7fff, + METHOD_TRANSFEROWNER = 0x8000, +}; + +enum EDecompressFlags +{ + DCF_TRANSFEROWNER = 1, + DCF_SEEKABLE = 2, + DCF_EXCEPTIONS = 4, + DCF_CACHED = 8, +}; + +bool OpenDecompressor(FileReader& self, FileReader &parent, FileReader::Size length, int method, int flags = 0); // creates a decompressor stream. 'seekable' uses a buffered version so that the Seek and Tell methods can be used. + +// This holds a compresed Zip entry with all needed info to decompress it. +struct FCompressedBuffer +{ + size_t mSize; + size_t mCompressedSize; + int mMethod; + unsigned mCRC32; + char* mBuffer; + const char* filename; + + bool Decompress(char* destbuffer); + void Clean() + { + mSize = mCompressedSize = 0; + if (mBuffer != nullptr) + { + delete[] mBuffer; + mBuffer = nullptr; + } + } +}; + + +} diff --git a/source/common/filesystem/include/fs_files.h b/source/common/filesystem/include/fs_files.h index 7a2871a71..463ede7ae 100644 --- a/source/common/filesystem/include/fs_files.h +++ b/source/common/filesystem/include/fs_files.h @@ -40,12 +40,11 @@ #include #include #include +#include #include #include #include "fs_swap.h" -#include "tarray.h" - namespace FileSys { class FileSystemException : public std::exception @@ -72,26 +71,10 @@ public: } }; -// Zip compression methods, extended by some internal types to be passed to OpenDecompressor -enum -{ - METHOD_STORED = 0, - METHOD_SHRINK = 1, - METHOD_IMPLODE = 6, - METHOD_DEFLATE = 8, - METHOD_BZIP2 = 12, - METHOD_LZMA = 14, - METHOD_XZ = 95, - METHOD_PPMD = 98, - METHOD_LZSS = 1337, // not used in Zips - this is for Console Doom compression - METHOD_ZLIB = 1338, // Zlib stream with header, used by compressed nodes. - METHOD_TRANSFEROWNER = 0x8000, -}; - class FileReader; // an opaque memory buffer to the file's content. Can either own the memory or just point to an external buffer. -class ResourceData +class FileData { void* memory; size_t length; @@ -99,13 +82,29 @@ class ResourceData public: using value_type = uint8_t; - ResourceData() { memory = nullptr; length = 0; owned = true; } + FileData() { memory = nullptr; length = 0; owned = true; } + FileData(const void* memory_, size_t len, bool own = true) + { + length = len; + if (own) + { + length = len; + memory = allocate(len); + if (memory_) memcpy(memory, memory_, len); + } + else + { + memory = (void*)memory_; + owned = false; + } + } + uint8_t* writable() const { return owned? (uint8_t*)memory : nullptr; } const void* data() const { return memory; } size_t size() const { return length; } const char* string() const { return (const char*)memory; } const uint8_t* bytes() const { return (const uint8_t*)memory; } - ResourceData& operator = (const ResourceData& copy) + FileData& operator = (const FileData& copy) { if (owned && memory) free(memory); length = copy.length; @@ -119,7 +118,7 @@ public: return *this; } - ResourceData& operator = (ResourceData&& copy) noexcept + FileData& operator = (FileData&& copy) noexcept { if (owned && memory) free(memory); length = copy.length; @@ -131,13 +130,13 @@ public: return *this; } - ResourceData(const ResourceData& copy) + FileData(const FileData& copy) { memory = nullptr; *this = copy; } - ~ResourceData() + ~FileData() { if (owned && memory) free(memory); } @@ -168,6 +167,7 @@ public: }; + class FileReaderInterface { public: @@ -181,12 +181,8 @@ public: ptrdiff_t GetLength () const { return Length; } }; -struct FResourceLump; - class FileReader { - friend struct FResourceLump; // needs access to the private constructor. - FileReaderInterface *mReader = nullptr; FileReader(const FileReader &r) = delete; @@ -210,13 +206,13 @@ public: FileReader() {} - FileReader(FileReader &&r) + FileReader(FileReader &&r) noexcept { mReader = r.mReader; r.mReader = nullptr; } - FileReader& operator =(FileReader &&r) + FileReader& operator =(FileReader &&r) noexcept { Close(); mReader = r.mReader; @@ -252,11 +248,7 @@ public: bool OpenFile(const char *filename, Size start = 0, Size length = -1, bool buffered = false); bool OpenFilePart(FileReader &parent, Size start, Size length); bool OpenMemory(const void *mem, Size length); // read directly from the buffer - bool OpenMemoryArray(const void *mem, Size length); // read from a copy of the buffer. - bool OpenMemoryArray(std::vector& data); // take the given array - bool OpenMemoryArray(ResourceData& data); // take the given array - bool OpenMemoryArray(std::function&)> getter); // read contents to a buffer and return a reader to it - bool OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions = false); // creates a decompressor stream. 'seekable' uses a buffered version so that the Seek and Tell methods can be used. + bool OpenMemoryArray(FileData& data); // take the given array Size Tell() const { @@ -273,36 +265,14 @@ public: return mReader->Read(buffer, len); } - ResourceData Read(size_t len) - { - ResourceData buffer; - if (len > 0) - { - Size length = mReader->Read(buffer.allocate(len), len); - if ((size_t)length < len) buffer.allocate(length); - } - return buffer; - } + FileData Read(size_t len); + FileData ReadPadded(size_t padding); - ResourceData Read() + FileData Read() { return Read(GetLength()); } - ResourceData ReadPadded(size_t padding) - { - auto len = GetLength(); - ResourceData buffer; - - if (len > 0) - { - auto p = (char*)buffer.allocate(len + padding); - Size length = mReader->Read(p, len); - if (length < len) buffer.clear(); - else memset(p + len, 0, padding); - } - return buffer; - } char *Gets(char *strbuf, Size len) { diff --git a/source/common/filesystem/include/fs_filesystem.h b/source/common/filesystem/include/fs_filesystem.h index 1b5249cbc..bfd134758 100644 --- a/source/common/filesystem/include/fs_filesystem.h +++ b/source/common/filesystem/include/fs_filesystem.h @@ -22,42 +22,6 @@ union LumpShortName }; -// A lump in memory. -class FileData -{ -public: - FileData() { lump = nullptr; } - const void *GetMem () { return lump->Cache; } - size_t GetSize () { return lump->LumpSize; } - const char* GetString () const { return (const char*)lump->Cache; } - const uint8_t* GetBytes() const { return (const uint8_t*)lump->Cache; } - - FileData& operator = (const FileData& copy) = delete; - - FileData(const FileData& copy) - { - lump = copy.lump; - lump->Lock(); - } - - ~FileData() - { - if (lump) lump->Unlock(); - } - - -private: - FileData(FResourceLump* nlump) - { - lump = nlump; - if (lump) lump->Lock(); - } - - FResourceLump* lump; - - friend class FileSystem; -}; - struct FolderEntry { const char *name; @@ -129,9 +93,19 @@ public: FileData ReadFile (const char *name) { return ReadFile (GetNumForName (name)); } FileData ReadFileFullName(const char* name) { return ReadFile(GetNumForFullName(name)); } - FileReader OpenFileReader(int lump); // opens a reader that redirects to the containing file's one. - FileReader ReopenFileReader(int lump, bool alwayscache = false); // opens an independent reader. + FileReader OpenFileReader(int lump, int readertype, int readerflags); // opens a reader that redirects to the containing file's one. FileReader OpenFileReader(const char* name); + FileReader ReopenFileReader(const char* name, bool alwayscache = false); + FileReader OpenFileReader(int lump) + { + return OpenFileReader(lump, READER_SHARED, READERFLAG_SEEKABLE); + } + + FileReader ReopenFileReader(int lump, bool alwayscache = false) + { + return OpenFileReader(lump, alwayscache ? READER_CACHED : READER_NEW, READERFLAG_SEEKABLE); + } + int FindLump (const char *name, int *lastlump, bool anyns=false); // [RH] Find lumps with duplication int FindLumpMulti (const char **names, int *lastlump, bool anyns = false, int *nameindex = NULL); // same with multiple possible names @@ -145,8 +119,7 @@ public: static uint32_t LumpNameHash (const char *name); // [RH] Create hash key from an 8-char name - int FileLength (int lump) const; - int GetFileOffset (int lump); // [RH] Returns offset of lump in the wadfile + ptrdiff_t FileLength (int lump) const; int GetFileFlags (int lump); // Return the flags for this lump const char* GetFileShortName(int lump) const; const char *GetFileFullName (int lump, bool returnshort = true) const; // [RH] Returns the lump's full name diff --git a/source/common/filesystem/include/fs_findfile.h b/source/common/filesystem/include/fs_findfile.h index d845b7d14..d8a18c374 100644 --- a/source/common/filesystem/include/fs_findfile.h +++ b/source/common/filesystem/include/fs_findfile.h @@ -23,7 +23,6 @@ using FileList = std::vector; struct FCompressedBuffer; bool ScanDirectory(std::vector& list, const char* dirpath, const char* match, bool nosubdir = false, bool readhidden = false); -bool WriteZip(const char* filename, const FCompressedBuffer* content, size_t contentcount); bool FS_DirEntryExists(const char* pathname, bool* isdir); inline void FixPathSeparator(char* path) diff --git a/source/common/filesystem/include/resourcefile.h b/source/common/filesystem/include/resourcefile.h index e2787fd12..20b48e0b5 100644 --- a/source/common/filesystem/include/resourcefile.h +++ b/source/common/filesystem/include/resourcefile.h @@ -7,6 +7,7 @@ #include #include #include "fs_files.h" +#include "fs_decompress.h" namespace FileSys { @@ -18,7 +19,6 @@ void strReplace(std::string& str, const char* from, const char* to); struct LumpFilterInfo { std::vector gameTypeFilter; // this can contain multiple entries - std::string dotFilter; // The following are for checking if the root directory of a zip can be removed. std::vector reservedFolders; @@ -78,125 +78,69 @@ typedef enum { enum ELumpFlags { - LUMPF_MAYBEFLAT = 1, // might be a flat outside F_START/END - LUMPF_FULLPATH = 2, // contains a full path. This will trigger extended namespace checks when looking up short names. - LUMPF_EMBEDDED = 4, // marks an embedded resource file for later processing. - LUMPF_SHORTNAME = 8, // the stored name is a short extension-less name - LUMPF_COMPRESSED = 16, // compressed or encrypted, i.e. cannot be read with the container file's reader. + RESFF_MAYBEFLAT = 1, // might be a flat inside a WAD outside F_START/END + RESFF_FULLPATH = 2, // contains a full path. This will trigger extended namespace checks when looking up short names. + RESFF_EMBEDDED = 4, // marks an embedded resource file for later processing. + RESFF_SHORTNAME = 8, // the stored name is a short extension-less name + RESFF_COMPRESSED = 16, // compressed or encrypted, i.e. cannot be read with the container file's reader. + RESFF_NEEDFILESTART = 32, // The real position is not known yet and needs to be calculated on access }; -// This holds a compresed Zip entry with all needed info to decompress it. -struct FCompressedBuffer +enum EReaderType { - size_t mSize; - size_t mCompressedSize; - int mMethod; - int mZipFlags; - unsigned mCRC32; - char *mBuffer; - const char* filename; - - bool Decompress(char *destbuffer); - void Clean() - { - mSize = mCompressedSize = 0; - if (mBuffer != nullptr) - { - delete[] mBuffer; - mBuffer = nullptr; - } - } + READER_SHARED = 0, // returns a view into the parent's reader. + READER_NEW = 1, // opens a new file handle + READER_CACHED = 2, // returns a MemoryArrayReader + READERFLAG_SEEKABLE = 1 // ensure the reader is seekable. }; -struct FResourceLump +struct FResourceEntry { -protected: - friend class FResourceFile; - friend class FWadFile; // this still needs direct access. - friend class FileData; - friend class FileSystem; - friend class FLumpFile; - friend class FLumpReader; - friend class FGrpFile; - friend class F7ZFile; - friend class FSSIFile; - friend class FWHResFile; - friend class FZipFile; - friend class FPakFile; - friend class FRFFFile; - friend class FDirectory; - friend int lumpcmp(const void* a, const void* b); - - - int LumpSize; - int RefCount; -//protected: - const char* FullName; -//public: - uint8_t Flags; - char * Cache; - FResourceFile * Owner; - -public: - FResourceLump() - { - Cache = NULL; - Owner = NULL; - Flags = 0; - RefCount = 0; - FullName = ""; - LumpSize = 0; - } - virtual ~FResourceLump(); - -protected: - - virtual FileReader *GetReader(); - virtual FileReader NewReader(); - virtual int GetFileOffset() { return -1; } - virtual int GetIndexNum() const { return -1; } - virtual int GetNamespace() const { return 0; } - void LumpNameSetup(const char* iname, StringPool* allocator); - void CheckEmbedded(LumpFilterInfo* lfi); - virtual FCompressedBuffer GetRawData(); - - void *Lock(); // validates the cache and increases the refcount. - int Unlock(); // decreases the refcount and frees the buffer - - unsigned Size() const{ return LumpSize; } - int LockCount() const { return RefCount; } - const char* getName() { return FullName; } - void clearName() { FullName = ""; } - -protected: - virtual int FillCache() { return -1; } - + size_t Length; + size_t CompressedSize; + const char* FileName; + size_t Position; + int ResourceID; + uint32_t CRC32; + uint16_t Flags; + uint16_t Method; + int16_t Namespace; }; +void SetMainThread(); + class FResourceFile { public: + FResourceFile(const char* filename, StringPool* sp); + FResourceFile(const char* filename, FileReader& r, StringPool* sp); + const char* NormalizeFileName(const char* fn, int fallbackcp = 0); + FResourceEntry* AllocateEntries(int count); + void GenerateHash(); + void PostProcessArchive(LumpFilterInfo* filter); protected: FileReader Reader; const char* FileName; + FResourceEntry* Entries = nullptr; uint32_t NumLumps; char Hash[48]; StringPool* stringpool; - FResourceFile(const char *filename, StringPool* sp); - FResourceFile(const char *filename, FileReader &r, StringPool* sp); - // for archives that can contain directories - void GenerateHash(); - void PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterInfo *filter); + virtual void SetEntryAddress(uint32_t entry) + { + Entries[entry].Flags &= ~RESFF_NEEDFILESTART; + } + bool IsFileInFolder(const char* const resPath); + void CheckEmbedded(uint32_t entry, LumpFilterInfo* lfi); private: uint32_t FirstLump; - int FilterLumps(const std::string& filtername, void *lumps, size_t lumpsize, uint32_t max); - int FilterLumpsByGameType(LumpFilterInfo *filter, void *lumps, size_t lumpsize, uint32_t max); - bool FindPrefixRange(const char* filter, void *lumps, size_t lumpsize, uint32_t max, uint32_t &start, uint32_t &end); - void JunkLeftoverFilters(void *lumps, size_t lumpsize, uint32_t max); + int FilterLumps(const std::string& filtername, uint32_t max); + bool FindPrefixRange(const char* filter, uint32_t max, uint32_t &start, uint32_t &end); + void JunkLeftoverFilters(uint32_t max); + void FindCommonFolder(LumpFilterInfo* filter); static FResourceFile *DoOpenResourceFile(const char *filename, FileReader &file, bool containeronly, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp); public: @@ -211,47 +155,45 @@ public: void SetFirstLump(uint32_t f) { FirstLump = f; } const char* GetHash() const { return Hash; } - virtual FResourceLump *GetLump(int no) = 0; - int EntryCount() const { return NumLumps; } int FindEntry(const char* name); - size_t Length(int entry) + size_t Length(uint32_t entry) { - auto l = GetLump(entry); - return l ? l->LumpSize : -1; + return (entry < NumLumps) ? Entries[entry].Length : 0; + } + size_t Offset(uint32_t entry) + { + return (entry < NumLumps) ? Entries[entry].Position : 0; } - FileReader GetEntryReader(int entry, bool newreader = true) + // default is the safest reader type. + virtual FileReader GetEntryReader(uint32_t entry, int readertype = READER_NEW, int flags = READERFLAG_SEEKABLE); + + int GetEntryFlags(uint32_t entry) { - auto l = GetLump(entry); - return l ? l->NewReader() : FileReader(); + return (entry < NumLumps) ? Entries[entry].Flags : 0; } - int GetEntryFlags(int entry) + int GetEntryNamespace(uint32_t entry) { - auto l = GetLump(entry); - return l ? l->Flags : 0; + return (entry < NumLumps) ? Entries[entry].Namespace : ns_hidden; } - ResourceData Read(int entry) + int GetEntryResourceID(uint32_t entry) { - auto fr = GetEntryReader(entry, false); - return fr.Read(); + return (entry < NumLumps) ? Entries[entry].ResourceID : -1; } - const char* getName(int entry) + const char* getName(uint32_t entry) { - auto l = GetLump(entry); - return l ? l->FullName : nullptr; - } - FCompressedBuffer GetRawData(int entry) - { - auto l = GetLump(entry); - if (!l) return {}; - return l->GetRawData(); + return (entry < NumLumps) ? Entries[entry].FileName : nullptr; } + virtual FileData Read(int entry); + + virtual FCompressedBuffer GetRawData(uint32_t entry); + FileReader Destroy() { auto fr = std::move(Reader); diff --git a/source/common/filesystem/source/w_zip.h b/source/common/filesystem/include/w_zip.h similarity index 99% rename from source/common/filesystem/source/w_zip.h rename to source/common/filesystem/include/w_zip.h index d26d7f003..50c39519f 100644 --- a/source/common/filesystem/source/w_zip.h +++ b/source/common/filesystem/include/w_zip.h @@ -7,7 +7,8 @@ #define FORCE_PACKED #endif -namespace FileSys { +#include + #pragma pack(1) // FZipCentralInfo @@ -107,5 +108,4 @@ struct FZipLocalFileHeader // File header flags. #define ZF_ENCRYPTED 0x1 -} #endif diff --git a/source/common/filesystem/source/ancientzip.cpp b/source/common/filesystem/source/ancientzip.cpp index 4d1e6b990..34356306e 100644 --- a/source/common/filesystem/source/ancientzip.cpp +++ b/source/common/filesystem/source/ancientzip.cpp @@ -44,6 +44,7 @@ #include +#include #include "ancientzip.h" namespace FileSys { @@ -125,7 +126,7 @@ static const unsigned char BitReverse4[] = { #define FIRST_BIT_LEN 8 #define REST_BIT_LEN 4 -void FZipExploder::InsertCode(TArray &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value) +void FZipExploder::InsertCode(std::vector &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value) { assert(len > 0); unsigned int node = pos + (code & ((1 << bits) - 1)); @@ -161,12 +162,12 @@ void FZipExploder::InsertCode(TArray &decoder, unsigned int pos, int b } } -unsigned int FZipExploder::InitTable(TArray &decoder, int numspots) +unsigned int FZipExploder::InitTable(std::vector &decoder, int numspots) { - unsigned int start = decoder.Size(); - decoder.Reserve(numspots); + size_t start = decoder.size(); + decoder.resize(decoder.size() + numspots); memset(&decoder[start], 0, sizeof(HuffNode)*numspots); - return start; + return (unsigned)start; } int FZipExploder::buildercmp(const void *a, const void *b) @@ -180,7 +181,7 @@ int FZipExploder::buildercmp(const void *a, const void *b) return d; } -int FZipExploder::BuildDecoder(TArray &decoder, TableBuilder *values, int numvals) +int FZipExploder::BuildDecoder(std::vector &decoder, TableBuilder *values, int numvals) { int i; @@ -218,7 +219,7 @@ int FZipExploder::BuildDecoder(TArray &decoder, TableBuilder *values, } -int FZipExploder::DecodeSFValue(const TArray &decoder) +int FZipExploder::DecodeSFValue(const std::vector &decoder) { unsigned int bits = FIRST_BIT_LEN, table = 0, code; const HuffNode *pos; @@ -236,7 +237,7 @@ int FZipExploder::DecodeSFValue(const TArray &decoder) } -int FZipExploder::DecodeSF(TArray &decoder, int numvals) +int FZipExploder::DecodeSF(std::vector &decoder, int numvals) { TableBuilder builder[256]; unsigned char a, c; diff --git a/source/common/filesystem/source/ancientzip.h b/source/common/filesystem/source/ancientzip.h index e25a75d3b..1f8d793fd 100644 --- a/source/common/filesystem/source/ancientzip.h +++ b/source/common/filesystem/source/ancientzip.h @@ -27,18 +27,18 @@ class FZipExploder unsigned short Code; }; - TArray LiteralDecoder; - TArray DistanceDecoder; - TArray LengthDecoder; + std::vector LiteralDecoder; + std::vector DistanceDecoder; + std::vector LengthDecoder; unsigned char ReadBuf[256]; unsigned int bs, be; static int buildercmp(const void *a, const void *b); - void InsertCode(TArray &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value); - unsigned int InitTable(TArray &decoder, int numspots); - int BuildDecoder(TArray &decoder, TableBuilder *values, int numvals); - int DecodeSFValue(const TArray ¤tTree); - int DecodeSF(TArray &decoder, int numvals); + void InsertCode(std::vector &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value); + unsigned int InitTable(std::vector &decoder, int numspots); + int BuildDecoder(std::vector &decoder, TableBuilder *values, int numvals); + int DecodeSFValue(const std::vector ¤tTree); + int DecodeSF(std::vector &decoder, int numvals); public: int Explode(unsigned char *out, unsigned int outsize, FileReader &in, unsigned int insize, int flags); }; diff --git a/source/common/filesystem/source/critsec.cpp b/source/common/filesystem/source/critsec.cpp new file mode 100644 index 000000000..874ade79f --- /dev/null +++ b/source/common/filesystem/source/critsec.cpp @@ -0,0 +1,150 @@ +/* +** +** +**--------------------------------------------------------------------------- +** Copyright 2005-2016 Randy Heit +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** 3. The name of the author may not be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**--------------------------------------------------------------------------- +** +*/ + +namespace FileSys { + +#ifdef _WIN32 + +#ifndef _WINNT_ +#define WIN32_LEAN_AND_MEAN +#include +#endif + +class FInternalCriticalSection +{ +public: + void Enter() + { + AcquireSRWLockExclusive(&CritSec); + } + void Leave() + { + ReleaseSRWLockExclusive(&CritSec); + } +private: + SRWLOCK CritSec = SRWLOCK_INIT; +}; + + +FInternalCriticalSection *CreateCriticalSection() +{ + return new FInternalCriticalSection(); +} + +void DeleteCriticalSection(FInternalCriticalSection *c) +{ + delete c; +} + +void EnterCriticalSection(FInternalCriticalSection *c) +{ + c->Enter(); +} + +void LeaveCriticalSection(FInternalCriticalSection *c) +{ + c->Leave(); +} + +#else + +#include "critsec.h" + +#include + +class FInternalCriticalSection +{ +public: + FInternalCriticalSection(); + ~FInternalCriticalSection(); + + void Enter(); + void Leave(); + +private: + pthread_mutex_t m_mutex; + +}; + +// TODO: add error handling + +FInternalCriticalSection::FInternalCriticalSection() +{ + pthread_mutexattr_t attributes; + pthread_mutexattr_init(&attributes); + pthread_mutexattr_settype(&attributes, PTHREAD_MUTEX_RECURSIVE); + + pthread_mutex_init(&m_mutex, &attributes); + + pthread_mutexattr_destroy(&attributes); +} + +FInternalCriticalSection::~FInternalCriticalSection() +{ + pthread_mutex_destroy(&m_mutex); +} + +void FInternalCriticalSection::Enter() +{ + pthread_mutex_lock(&m_mutex); +} + +void FInternalCriticalSection::Leave() +{ + pthread_mutex_unlock(&m_mutex); +} + + +FInternalCriticalSection *CreateCriticalSection() +{ + return new FInternalCriticalSection(); +} + +void DeleteCriticalSection(FInternalCriticalSection *c) +{ + delete c; +} + +void EnterCriticalSection(FInternalCriticalSection *c) +{ + c->Enter(); +} + +void LeaveCriticalSection(FInternalCriticalSection *c) +{ + c->Leave(); +} + +#endif + +} \ No newline at end of file diff --git a/source/common/filesystem/source/critsec.h b/source/common/filesystem/source/critsec.h new file mode 100644 index 000000000..9ea56aab5 --- /dev/null +++ b/source/common/filesystem/source/critsec.h @@ -0,0 +1,39 @@ +#pragma once + +namespace FileSys { +// System independent critical sections without polluting the namespace with the operating system headers. +class FInternalCriticalSection; +FInternalCriticalSection *CreateCriticalSection(); +void DeleteCriticalSection(FInternalCriticalSection *c); +void EnterCriticalSection(FInternalCriticalSection *c); +void LeaveCriticalSection(FInternalCriticalSection *c); + +// This is just a convenience wrapper around the function interface adjusted to use std::lock_guard +class FCriticalSection +{ +public: + FCriticalSection() + { + c = CreateCriticalSection(); + } + + ~FCriticalSection() + { + DeleteCriticalSection(c); + } + + void lock() + { + EnterCriticalSection(c); + } + + void unlock() + { + LeaveCriticalSection(c); + } + +private: + FInternalCriticalSection *c; + +}; +} \ No newline at end of file diff --git a/source/common/filesystem/source/file_7z.cpp b/source/common/filesystem/source/file_7z.cpp index 0740ceca1..7efca8c9d 100644 --- a/source/common/filesystem/source/file_7z.cpp +++ b/source/common/filesystem/source/file_7z.cpp @@ -38,6 +38,9 @@ #include "7zCrc.h" #include "resourcefile.h" #include "fs_findfile.h" +#include "unicode.h" +#include "critsec.h" +#include namespace FileSys { @@ -158,20 +161,6 @@ struct C7zArchive return res; } }; -//========================================================================== -// -// Zip Lump -// -//========================================================================== - -struct F7ZLump : public FResourceLump -{ - int Position; - - virtual int FillCache() override; - -}; - //========================================================================== // @@ -183,14 +172,15 @@ class F7ZFile : public FResourceFile { friend struct F7ZLump; - F7ZLump *Lumps; C7zArchive *Archive; + FCriticalSection critsec; public: F7ZFile(const char * filename, FileReader &filer, StringPool* sp); bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf); virtual ~F7ZFile(); - virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; } + FileData Read(int entry) override; + FileReader GetEntryReader(uint32_t entry, int, int) override; }; @@ -204,8 +194,7 @@ public: F7ZFile::F7ZFile(const char * filename, FileReader &filer, StringPool* sp) : FResourceFile(filename, filer, sp) { - Lumps = NULL; - Archive = NULL; + Archive = nullptr; } @@ -247,12 +236,11 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf) CSzArEx* const archPtr = &Archive->DB; + AllocateEntries(archPtr->NumFiles); NumLumps = archPtr->NumFiles; - Lumps = new F7ZLump[NumLumps]; - F7ZLump *lump_p = Lumps; std::u16string nameUTF16; - std::string nameASCII; + std::vector nameASCII; for (uint32_t i = 0; i < NumLumps; ++i) { @@ -273,21 +261,17 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf) nameUTF16.resize((unsigned)nameLength); nameASCII.resize((unsigned)nameLength); - // note that the file system is not equipped to handle non-ASCII, so don't bother with proper Unicode conversion here. - SzArEx_GetFileNameUtf16(archPtr, i, (UInt16*)nameUTF16.data()); - for (size_t c = 0; c < nameLength; ++c) - { - nameASCII[c] = tolower(static_cast(nameUTF16[c])); - } - FixPathSeparator(&nameASCII.front()); - lump_p->LumpNameSetup(nameASCII.c_str(), stringpool); - lump_p->LumpSize = static_cast(SzArEx_GetFileSize(archPtr, i)); - lump_p->Owner = this; - lump_p->Flags = LUMPF_FULLPATH|LUMPF_COMPRESSED; - lump_p->Position = i; - lump_p->CheckEmbedded(filter); - lump_p++; + SzArEx_GetFileNameUtf16(archPtr, i, (UInt16*)nameUTF16.data()); + utf16_to_utf8((uint16_t*)nameUTF16.data(), nameASCII); + + Entries[i].FileName = NormalizeFileName(nameASCII.data()); + Entries[i].Length = SzArEx_GetFileSize(archPtr, i); + Entries[i].Flags = RESFF_FULLPATH|RESFF_COMPRESSED; + Entries[i].ResourceID = -1; + Entries[i].Namespace = ns_global; + Entries[i].Method = METHOD_INVALID; + Entries[i].Position = i; } // Resize the lump record array to its actual size NumLumps -= skipped; @@ -296,10 +280,9 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf) { // Quick check for unsupported compression method - TArray temp; - temp.Resize(Lumps[0].LumpSize); + FileData temp(nullptr, Entries[0].Length); - if (SZ_OK != Archive->Extract(Lumps[0].Position, &temp[0])) + if (SZ_OK != Archive->Extract(Entries[0].Position, (char*)temp.writable())) { Printf(FSMessageLevel::Error, "%s: unsupported 7z/LZMA file!\n", FileName); return false; @@ -307,7 +290,7 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf) } GenerateHash(); - PostProcessArchive(&Lumps[0], sizeof(F7ZLump), filter); + PostProcessArchive(filter); return true; } @@ -319,11 +302,7 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf) F7ZFile::~F7ZFile() { - if (Lumps != NULL) - { - delete[] Lumps; - } - if (Archive != NULL) + if (Archive != nullptr) { delete Archive; } @@ -331,22 +310,41 @@ F7ZFile::~F7ZFile() //========================================================================== // -// Fills the lump cache and performs decompression +// Reads data for one entry into a buffer // //========================================================================== -int F7ZLump::FillCache() +FileData F7ZFile::Read(int entry) { - Cache = new char[LumpSize]; - SRes code = static_cast(Owner)->Archive->Extract(Position, Cache); - if (code != SZ_OK) + FileData buffer; + if ((entry >= 0 || entry < NumLumps) && Entries[entry].Length > 0) { - throw FileSystemException("Error %d reading from 7z archive", code); + auto p = buffer.allocate(Entries[entry].Length); + // There is no realistic way to keep multiple references to a 7z file open without massive overhead so to make this thread-safe a mutex is the only option. + std::lock_guard lock(critsec); + SRes code = Archive->Extract(Entries[entry].Position, (char*)p); + if (code != SZ_OK) buffer.clear(); } - RefCount = 1; - return 1; + return buffer; } +//========================================================================== +// +// This can only return a FileReader to a memory buffer. +// +//========================================================================== + +FileReader F7ZFile::GetEntryReader(uint32_t entry, int, int) +{ + FileReader fr; + if (entry < 0 || entry >= NumLumps) return fr; + auto buffer = Read(entry); + if (buffer.size() > 0) + fr.OpenMemoryArray(buffer); + return fr; +} + + //========================================================================== // // File open diff --git a/source/common/filesystem/source/file_directory.cpp b/source/common/filesystem/source/file_directory.cpp index 89b9625fc..0b9c101ce 100644 --- a/source/common/filesystem/source/file_directory.cpp +++ b/source/common/filesystem/source/file_directory.cpp @@ -48,21 +48,6 @@ std::string FS_FullPath(const char* directory); std::wstring toWide(const char* str); #endif -//========================================================================== -// -// Zip Lump -// -//========================================================================== - -struct FDirectoryLump : public FResourceLump -{ - FileReader NewReader() override; - int FillCache() override; - - const char* mFullPath; -}; - - //========================================================================== // // Zip file @@ -71,16 +56,15 @@ struct FDirectoryLump : public FResourceLump class FDirectory : public FResourceFile { - TArray Lumps; const bool nosubdir; + const char* mBasePath; int AddDirectory(const char* dirpath, LumpFilterInfo* filter, FileSystemMessageFunc Printf); - void AddEntry(const char *fullpath, const char* relpath, int size); public: FDirectory(const char * dirname, StringPool* sp, bool nosubdirflag = false); bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf); - virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; } + FileReader GetEntryReader(uint32_t entry, int, int) override; }; @@ -116,8 +100,17 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy } else { + mBasePath = nullptr; + AllocateEntries(list.size()); for(auto& entry : list) { + if (mBasePath == nullptr) + { + // extract the base path from the first entry to cover changes made in ScanDirectory. + auto full = entry.FilePath.find(entry.FilePathRel); + std::string path(entry.FilePath, 0, full); + mBasePath = stringpool->Strdup(path.c_str()); + } if (!entry.isDirectory) { auto fi = entry.FileName; @@ -128,6 +121,7 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy continue; } + if (filter->filenamecheck == nullptr || filter->filenamecheck(fi.c_str(), entry.FilePath.c_str())) { if (entry.Length > 0x7fffffff) @@ -135,7 +129,13 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy Printf(FSMessageLevel::Warning, "%s is larger than 2GB and will be ignored\n", entry.FilePath.c_str()); continue; } - AddEntry(entry.FilePath.c_str(), entry.FilePathRel.c_str(), (int)entry.Length); + // for internal access we use the normalized form of the relative path. + Entries[count].FileName = NormalizeFileName(entry.FilePathRel.c_str()); + Entries[count].Length = entry.Length; + Entries[count].Flags = RESFF_FULLPATH; + Entries[count].ResourceID = -1; + Entries[count].Method = METHOD_STORED; + Entries[count].Namespace = ns_global; count++; } } @@ -153,7 +153,6 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy bool FDirectory::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) { NumLumps = AddDirectory(FileName, filter, Printf); - PostProcessArchive(&Lumps[0], sizeof(FDirectoryLump), filter); return true; } @@ -163,62 +162,22 @@ bool FDirectory::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) // //========================================================================== -void FDirectory::AddEntry(const char *fullpath, const char* relpath, int size) -{ - FDirectoryLump *lump_p = &Lumps[Lumps.Reserve(1)]; - - // Store the full path here so that we can access the file later, even if it is from a filter directory. - lump_p->mFullPath = stringpool->Strdup(fullpath); - - // [mxd] Convert name to lowercase - std::string name = relpath; - for (auto& c : name) c = tolower(c); - - // The lump's name is only the part relative to the main directory - lump_p->LumpNameSetup(name.c_str(), stringpool); - lump_p->LumpSize = size; - lump_p->Owner = this; - lump_p->Flags = 0; - lump_p->CheckEmbedded(nullptr); -} - - -//========================================================================== -// -// -// -//========================================================================== - -FileReader FDirectoryLump::NewReader() +FileReader FDirectory::GetEntryReader(uint32_t entry, int readertype, int) { FileReader fr; - fr.OpenFile(mFullPath); + if (entry < NumLumps) + { + std::string fn = mBasePath; fn += Entries[entry].FileName; + fr.OpenFile(fn.c_str()); + if (readertype == READER_CACHED) + { + auto data = fr.Read(); + fr.OpenMemoryArray(data); + } + } return fr; } -//========================================================================== -// -// -// -//========================================================================== - -int FDirectoryLump::FillCache() -{ - FileReader fr; - Cache = new char[LumpSize]; - if (!fr.OpenFile(mFullPath)) - { - throw FileSystemException("unable to open file"); - } - auto read = fr.Read(Cache, LumpSize); - if (read != LumpSize) - { - throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize); - } - RefCount = 1; - return 1; -} - //========================================================================== // // File open diff --git a/source/common/filesystem/source/file_grp.cpp b/source/common/filesystem/source/file_grp.cpp index 05299f940..39d8abc19 100644 --- a/source/common/filesystem/source/file_grp.cpp +++ b/source/common/filesystem/source/file_grp.cpp @@ -33,7 +33,7 @@ ** */ -#include "resourcefile_internal.h" +#include "resourcefile.h" #include "fs_swap.h" namespace FileSys { @@ -65,62 +65,39 @@ struct GrpLump }; -//========================================================================== -// -// Build GRP file -// -//========================================================================== - -class FGrpFile : public FUncompressedFile -{ -public: - FGrpFile(const char * filename, FileReader &file, StringPool* sp); - bool Open(LumpFilterInfo* filter); -}; - - -//========================================================================== -// -// Initializes a Build GRP file -// -//========================================================================== - -FGrpFile::FGrpFile(const char *filename, FileReader &file, StringPool* sp) -: FUncompressedFile(filename, file, sp) -{ -} - //========================================================================== // // Open it // //========================================================================== -bool FGrpFile::Open(LumpFilterInfo* filter) +static bool OpenGrp(FResourceFile* file, LumpFilterInfo* filter) { GrpHeader header; - Reader.Read(&header, sizeof(header)); - NumLumps = LittleLong(header.NumLumps); + auto Reader = file->GetContainerReader(); + Reader->Read(&header, sizeof(header)); + uint32_t NumLumps = LittleLong(header.NumLumps); + auto Entries = file->AllocateEntries(NumLumps); GrpLump *fileinfo = new GrpLump[NumLumps]; - Reader.Read (fileinfo, NumLumps * sizeof(GrpLump)); - - Lumps.Resize(NumLumps); + Reader->Read (fileinfo, NumLumps * sizeof(GrpLump)); int Position = sizeof(GrpHeader) + NumLumps * sizeof(GrpLump); for(uint32_t i = 0; i < NumLumps; i++) { - Lumps[i].Owner = this; - Lumps[i].Position = Position; - Lumps[i].LumpSize = LittleLong(fileinfo[i].Size); + Entries[i].Position = Position; + Entries[i].Length = LittleLong(fileinfo[i].Size); Position += fileinfo[i].Size; - Lumps[i].Flags = 0; + Entries[i].Flags = 0; + Entries[i].Namespace = ns_global; fileinfo[i].NameWithZero[12] = '\0'; // Be sure filename is null-terminated - Lumps[i].LumpNameSetup(fileinfo[i].NameWithZero, stringpool); + Entries[i].ResourceID = -1; + Entries[i].Method = METHOD_STORED; + Entries[i].FileName = file->NormalizeFileName(fileinfo[i].Name); } - GenerateHash(); + file->GenerateHash(); delete[] fileinfo; return true; } @@ -143,12 +120,12 @@ FResourceFile *CheckGRP(const char *filename, FileReader &file, LumpFilterInfo* file.Seek(0, FileReader::SeekSet); if (!memcmp(head, "KenSilverman", 12)) { - auto rf = new FGrpFile(filename, file, sp); - if (rf->Open(filter)) return rf; + auto rf = new FResourceFile(filename, file, sp); + if (OpenGrp(rf, filter)) return rf; file = rf->Destroy(); } } - return NULL; + return nullptr; } } diff --git a/source/common/filesystem/source/file_hog.cpp b/source/common/filesystem/source/file_hog.cpp new file mode 100644 index 000000000..cc4eef4ff --- /dev/null +++ b/source/common/filesystem/source/file_hog.cpp @@ -0,0 +1,109 @@ +/* +** file_hog.cpp +** +** reads Descent .hog files +** +**--------------------------------------------------------------------------- +** Copyright 2023 Christoph Oelckers +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** 3. The name of the author may not be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**--------------------------------------------------------------------------- +** +** +*/ + + +#include "resourcefile.h" +#include "fs_swap.h" + +namespace FileSys { + using namespace byteswap; + + + + +static bool OpenHog(FResourceFile* rf, LumpFilterInfo* filter) +{ + auto Reader = rf->GetContainerReader(); + FileReader::Size length = Reader->GetLength(); + + std::vector entries; + // Hogs store their data as a list of file records, each containing a name, length and the actual data. + // To read the directory the entire file must be scanned. + while (Reader->Tell() <= length) + { + char name[13]; + + auto r = Reader->Read(&name, 13); + if (r < 13) break; + name[12] = 0; + uint32_t elength = Reader->ReadUInt32(); + + FResourceEntry Entry; + Entry.Position = Reader->Tell(); + Entry.Length = elength; + Entry.Flags = 0; + Entry.CRC32 = 0; + Entry.Namespace = ns_global; + Entry.ResourceID = -1; + Entry.Method = METHOD_STORED; + Entry.FileName = rf->NormalizeFileName(name); + entries.push_back(Entry); + Reader->Seek(elength, FileReader::SeekCur); + } + auto Entries = rf->AllocateEntries((int)entries.size()); + memcpy(Entries, entries.data(), entries.size() * sizeof(Entries[0])); + rf->GenerateHash(); + return true; +} + + +//========================================================================== +// +// File open +// +//========================================================================== + +FResourceFile* CheckHog(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp) +{ + char head[3]; + + if (file.GetLength() >= 20) + { + file.Seek(0, FileReader::SeekSet); + file.Read(&head, 3); + if (!memcmp(head, "DHF", 3)) + { + auto rf = new FResourceFile(filename, file, sp); + if (OpenHog(rf, filter)) return rf; + file = rf->Destroy(); + } + file.Seek(0, FileReader::SeekSet); + } + return nullptr; +} + + +} \ No newline at end of file diff --git a/source/common/filesystem/source/file_lump.cpp b/source/common/filesystem/source/file_lump.cpp index 94da2af4f..b08f3bf65 100644 --- a/source/common/filesystem/source/file_lump.cpp +++ b/source/common/filesystem/source/file_lump.cpp @@ -32,49 +32,25 @@ ** */ -#include "resourcefile_internal.h" +#include "resourcefile.h" namespace FileSys { -//========================================================================== -// -// Single lump -// -//========================================================================== - -class FLumpFile : public FUncompressedFile -{ -public: - FLumpFile(const char * filename, FileReader &file, StringPool* sp); - bool Open(LumpFilterInfo* filter); -}; - - -//========================================================================== -// -// FLumpFile::FLumpFile -// -//========================================================================== - -FLumpFile::FLumpFile(const char *filename, FileReader &file, StringPool* sp) - : FUncompressedFile(filename, file, sp) -{ -} - //========================================================================== // // Open it // //========================================================================== -bool FLumpFile::Open(LumpFilterInfo*) +static bool OpenLump(FResourceFile* file, LumpFilterInfo*) { - Lumps.Resize(1); - Lumps[0].LumpNameSetup(ExtractBaseName(FileName, true).c_str(), stringpool); - Lumps[0].Owner = this; - Lumps[0].Position = 0; - Lumps[0].LumpSize = (int)Reader.GetLength(); - Lumps[0].Flags = 0; - NumLumps = 1; + auto Entries = file->AllocateEntries(1); + Entries[0].FileName = file->NormalizeFileName(ExtractBaseName(file->GetFileName(), true).c_str()); + Entries[0].Namespace = ns_global; + Entries[0].ResourceID = -1; + Entries[0].Position = 0; + Entries[0].Length = file->GetContainerReader()->GetLength(); + Entries[0].Method = METHOD_STORED; + Entries[0].Flags = 0; return true; } @@ -87,8 +63,8 @@ bool FLumpFile::Open(LumpFilterInfo*) FResourceFile *CheckLump(const char *filename, FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp) { // always succeeds - auto rf = new FLumpFile(filename, file, sp); - if (rf->Open(filter)) return rf; + auto rf = new FResourceFile(filename, file, sp); + if (OpenLump(rf, filter)) return rf; file = rf->Destroy(); return NULL; } diff --git a/source/common/filesystem/source/file_mvl.cpp b/source/common/filesystem/source/file_mvl.cpp new file mode 100644 index 000000000..48babef16 --- /dev/null +++ b/source/common/filesystem/source/file_mvl.cpp @@ -0,0 +1,98 @@ +/* +** file_mvl.cpp +** +** reads Descent2 .mvl files +** +**--------------------------------------------------------------------------- +** Copyright 2023 Christoph Oelckers +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** 3. The name of the author may not be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**--------------------------------------------------------------------------- +** +** +*/ + + +#include "resourcefile.h" +#include "fs_swap.h" + +namespace FileSys { + using namespace byteswap; + + + +static bool OpenMvl(FResourceFile* rf, LumpFilterInfo* filter) +{ + auto Reader = rf->GetContainerReader(); + auto count = Reader->ReadUInt32(); + auto Entries = rf->AllocateEntries(count); + size_t pos = 8 + (17 * count); // files start after the directory + + for (uint32_t i = 0; i < count; i++) + { + char name[13]; + Reader->Read(&name, 13); + name[12] = 0; + uint32_t elength = Reader->ReadUInt32(); + + Entries[i].Position = pos; + Entries[i].Length = elength; + Entries[i].ResourceID = -1; + Entries[i].FileName = rf->NormalizeFileName(name); + + pos += elength; + } + + return true; +} + + +//========================================================================== +// +// File open +// +//========================================================================== + +FResourceFile* CheckMvl(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp) +{ + char head[4]; + + if (file.GetLength() >= 20) + { + file.Seek(0, FileReader::SeekSet); + file.Read(&head, 4); + if (!memcmp(head, "DMVL", 4)) + { + auto rf = new FResourceFile(filename, file, sp); + if (OpenMvl(rf, filter)) return rf; + file = rf->Destroy(); + } + file.Seek(0, FileReader::SeekSet); + } + return nullptr; +} + + +} \ No newline at end of file diff --git a/source/common/filesystem/source/file_pak.cpp b/source/common/filesystem/source/file_pak.cpp index 12ab2f4e2..c1d559eb9 100644 --- a/source/common/filesystem/source/file_pak.cpp +++ b/source/common/filesystem/source/file_pak.cpp @@ -32,7 +32,7 @@ ** */ -#include "resourcefile_internal.h" +#include "resourcefile.h" namespace FileSys { @@ -57,64 +57,38 @@ struct dpackheader_t } ; -//========================================================================== -// -// Wad file -// -//========================================================================== - -class FPakFile : public FUncompressedFile -{ -public: - FPakFile(const char * filename, FileReader &file, StringPool* sp); - bool Open(LumpFilterInfo* filter); -}; - - -//========================================================================== -// -// FWadFile::FWadFile -// -// Initializes a WAD file -// -//========================================================================== - -FPakFile::FPakFile(const char *filename, FileReader &file, StringPool* sp) - : FUncompressedFile(filename, file, sp) -{ -} - //========================================================================== // // Open it // //========================================================================== -bool FPakFile::Open(LumpFilterInfo* filter) +static bool OpenPak(FResourceFile* file, LumpFilterInfo* filter) { dpackheader_t header; - Reader.Read(&header, sizeof(header)); - NumLumps = LittleLong(header.dirlen) / sizeof(dpackfile_t); + auto Reader = file->GetContainerReader(); + Reader->Read(&header, sizeof(header)); + uint32_t NumLumps = header.dirlen / sizeof(dpackfile_t); + auto Entries = file->AllocateEntries(NumLumps); header.dirofs = LittleLong(header.dirofs); - TArray fileinfo(NumLumps, true); - Reader.Seek (header.dirofs, FileReader::SeekSet); - Reader.Read (fileinfo.Data(), NumLumps * sizeof(dpackfile_t)); - - Lumps.Resize(NumLumps); + Reader->Seek (header.dirofs, FileReader::SeekSet); + auto fd = Reader->Read (NumLumps * sizeof(dpackfile_t)); + auto fileinfo = (const dpackfile_t*)fd.data(); for(uint32_t i = 0; i < NumLumps; i++) { - Lumps[i].LumpNameSetup(fileinfo[i].name, stringpool); - Lumps[i].Flags = LUMPF_FULLPATH; - Lumps[i].Owner = this; - Lumps[i].Position = LittleLong(fileinfo[i].filepos); - Lumps[i].LumpSize = LittleLong(fileinfo[i].filelen); - Lumps[i].CheckEmbedded(filter); + Entries[i].Position = LittleLong(fileinfo[i].filepos); + Entries[i].Length = LittleLong(fileinfo[i].filelen); + Entries[i].Flags = RESFF_FULLPATH; + Entries[i].Namespace = ns_global; + Entries[i].ResourceID = -1; + Entries[i].Method = METHOD_STORED; + Entries[i].FileName = file->NormalizeFileName(fileinfo[i].name); } - GenerateHash(); - PostProcessArchive(&Lumps[0], sizeof(Lumps[0]), filter); + file->GenerateHash(); + file->PostProcessArchive(filter); return true; } @@ -136,8 +110,8 @@ FResourceFile *CheckPak(const char *filename, FileReader &file, LumpFilterInfo* file.Seek(0, FileReader::SeekSet); if (!memcmp(head, "PACK", 4)) { - auto rf = new FPakFile(filename, file, sp); - if (rf->Open(filter)) return rf; + auto rf = new FResourceFile(filename, file, sp); + if (OpenPak(rf, filter)) return rf; file = rf->Destroy(); } } diff --git a/source/common/filesystem/source/file_rff.cpp b/source/common/filesystem/source/file_rff.cpp index dff7b69e9..a3545d3b9 100644 --- a/source/common/filesystem/source/file_rff.cpp +++ b/source/common/filesystem/source/file_rff.cpp @@ -33,7 +33,8 @@ ** */ -#include "resourcefile_internal.h" +#include +#include "resourcefile.h" #include "fs_swap.h" namespace FileSys { @@ -67,22 +68,6 @@ struct RFFLump uint32_t IndexNum; // Used by .sfx, possibly others }; -//========================================================================== -// -// Blood RFF lump (uncompressed lump with encryption) -// -//========================================================================== - -struct FRFFLump : public FUncompressedLump -{ - virtual FileReader *GetReader(); - virtual int FillCache() override; - - uint32_t IndexNum; - - int GetIndexNum() const { return IndexNum; } -}; - //========================================================================== // // BloodCrypt @@ -100,68 +85,47 @@ void BloodCrypt (void *data, int key, int len) } -//========================================================================== -// -// Blood RFF file -// -//========================================================================== - -class FRFFFile : public FResourceFile -{ - FRFFLump *Lumps; - -public: - FRFFFile(const char * filename, FileReader &file, StringPool* sp); - virtual ~FRFFFile(); - virtual bool Open(LumpFilterInfo* filter); - virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; } -}; - - //========================================================================== // // Initializes a Blood RFF file // //========================================================================== -FRFFFile::FRFFFile(const char *filename, FileReader &file, StringPool* sp) -: FResourceFile(filename, file, sp) -{ - Lumps = NULL; -} - -//========================================================================== -// -// Initializes a Blood RFF file -// -//========================================================================== - -bool FRFFFile::Open(LumpFilterInfo*) +static bool OpenRFF(FResourceFile* file, LumpFilterInfo*) { RFFLump *lumps; RFFInfo header; - Reader.Read(&header, sizeof(header)); + auto Reader = file->GetContainerReader(); + Reader->Read(&header, sizeof(header)); - NumLumps = LittleLong(header.NumLumps); + uint32_t NumLumps = LittleLong(header.NumLumps); + auto Entries = file->AllocateEntries(NumLumps); header.DirOfs = LittleLong(header.DirOfs); lumps = new RFFLump[header.NumLumps]; - Reader.Seek (header.DirOfs, FileReader::SeekSet); - Reader.Read (lumps, header.NumLumps * sizeof(RFFLump)); - BloodCrypt (lumps, header.DirOfs, header.NumLumps * sizeof(RFFLump)); - - Lumps = new FRFFLump[NumLumps]; + Reader->Seek (LittleLong(header.DirOfs), FileReader::SeekSet); + Reader->Read (lumps, NumLumps * sizeof(RFFLump)); + BloodCrypt (lumps, LittleLong(header.DirOfs), NumLumps * sizeof(RFFLump)); for (uint32_t i = 0; i < NumLumps; ++i) { - Lumps[i].Position = LittleLong(lumps[i].FilePos); - Lumps[i].LumpSize = LittleLong(lumps[i].Size); - Lumps[i].Owner = this; + Entries[i].Position = LittleLong(lumps[i].FilePos); + Entries[i].Length = LittleLong(lumps[i].Size); + Entries[i].Flags = 0; + Entries[i].Method = METHOD_STORED; if (lumps[i].Flags & 0x10) { - Lumps[i].Flags |= LUMPF_COMPRESSED; // flags the lump as not directly usable + Entries[i].Flags = RESFF_COMPRESSED; // for purposes of decoding, compression and encryption are equivalent. + Entries[i].Method = METHOD_RFFCRYPT; } - Lumps[i].IndexNum = LittleLong(lumps[i].IndexNum); + else + { + Entries[i].Flags = 0; + Entries[i].Method = METHOD_STORED; + } + Entries[i].Namespace = ns_global; + Entries[i].ResourceID = LittleLong(lumps[i].IndexNum); + // Rearrange the name and extension to construct the fullname. char name[13]; strncpy(name, lumps[i].Name, 8); @@ -173,66 +137,13 @@ bool FRFFFile::Open(LumpFilterInfo*) name[len+2] = lumps[i].Extension[1]; name[len+3] = lumps[i].Extension[2]; name[len+4] = 0; - Lumps[i].LumpNameSetup(name, stringpool); + Entries[i].FileName = file->NormalizeFileName(name); } delete[] lumps; - GenerateHash(); + file->GenerateHash(); return true; } -FRFFFile::~FRFFFile() -{ - if (Lumps != NULL) - { - delete[] Lumps; - } -} - - -//========================================================================== -// -// Get reader (only returns non-NULL if not encrypted) -// -//========================================================================== - -FileReader *FRFFLump::GetReader() -{ - // Don't return the reader if this lump is encrypted - // In that case always force caching of the lump - if (!(Flags & LUMPF_COMPRESSED)) - { - return FUncompressedLump::GetReader(); - } - else - { - return NULL; - } -} - -//========================================================================== -// -// Fills the lump cache and performs decryption -// -//========================================================================== - -int FRFFLump::FillCache() -{ - int res = FUncompressedLump::FillCache(); - - if (Flags & LUMPF_COMPRESSED) - { - int cryptlen = std::min (LumpSize, 256); - uint8_t *data = (uint8_t *)Cache; - - for (int i = 0; i < cryptlen; ++i) - { - data[i] ^= i >> 1; - } - } - return res; -} - - //========================================================================== // // File open @@ -250,8 +161,8 @@ FResourceFile *CheckRFF(const char *filename, FileReader &file, LumpFilterInfo* file.Seek(0, FileReader::SeekSet); if (!memcmp(head, "RFF\x1a", 4)) { - auto rf = new FRFFFile(filename, file, sp); - if (rf->Open(filter)) return rf; + auto rf = new FResourceFile(filename, file, sp); + if (OpenRFF(rf, filter)) return rf; file = rf->Destroy(); } } diff --git a/source/common/filesystem/source/file_ssi.cpp b/source/common/filesystem/source/file_ssi.cpp index f23c8e5ea..a5efec8c1 100644 --- a/source/common/filesystem/source/file_ssi.cpp +++ b/source/common/filesystem/source/file_ssi.cpp @@ -33,33 +33,9 @@ ** */ -#include "resourcefile_internal.h" +#include "resourcefile.h" namespace FileSys { -//========================================================================== -// -// Build GRP file -// -//========================================================================== - -class FSSIFile : public FUncompressedFile -{ -public: - FSSIFile(const char * filename, FileReader &file, StringPool* sp); - bool Open(int version, int EntryCount, LumpFilterInfo* filter); -}; - - -//========================================================================== -// -// Initializes a Build GRP file -// -//========================================================================== - -FSSIFile::FSSIFile(const char *filename, FileReader &file, StringPool* sp) -: FUncompressedFile(filename, file, sp) -{ -} //========================================================================== // @@ -68,42 +44,50 @@ FSSIFile::FSSIFile(const char *filename, FileReader &file, StringPool* sp) // //========================================================================== -bool FSSIFile::Open(int version, int EntryCount, LumpFilterInfo*) +static bool OpenSSI(FResourceFile* file, int version, int EntryCount, LumpFilterInfo*) { - NumLumps = EntryCount*2; - Lumps.Resize(EntryCount*2); - + uint32_t NumLumps = EntryCount * 2; + auto Entries = file->AllocateEntries(NumLumps); + auto Reader = file->GetContainerReader(); + int32_t j = (version == 2 ? 267 : 254) + (EntryCount * 121); for (uint32_t i = 0; i < NumLumps; i+=2) { char fn[13]; - int strlength = Reader.ReadUInt8(); + int strlength = Reader->ReadUInt8(); if (strlength > 12) strlength = 12; - Reader.Read(fn, 12); + Reader->Read(fn, 12); fn[strlength] = 0; - int flength = Reader.ReadInt32(); + int flength = Reader->ReadInt32(); - - Lumps[i].LumpNameSetup(fn, stringpool); - Lumps[i].Position = j; - Lumps[i].LumpSize = flength; - Lumps[i].Owner = this; - if (strstr(fn, ".GRP")) Lumps[i].Flags |= LUMPF_EMBEDDED; + Entries[i].Position = j; + Entries[i].Length = flength; + Entries[i].Flags = 0; + Entries[i].Namespace = ns_global; + Entries[i].Method = METHOD_STORED; + Entries[i].ResourceID = -1; + Entries[i].FileName = file->NormalizeFileName(fn); + if (strstr(fn, ".GRP")) Entries[i].Flags |= RESFF_EMBEDDED; // SSI files can swap the order of the extension's characters - but there's no reliable detection for this and it can be mixed inside the same container, // so we have no choice but to create another file record for the altered name. std::swap(fn[strlength - 1], fn[strlength - 3]); - Lumps[i+1].LumpNameSetup(fn, stringpool); - Lumps[i+1].Position = j; - Lumps[i+1].LumpSize = flength; - Lumps[i+1].Owner = this; - if (strstr(fn, ".GRP")) Lumps[i+1].Flags |= LUMPF_EMBEDDED; + + Entries[i + 1].Position = j; + Entries[i + 1].Length = flength; + Entries[i + 1].Flags = 0; + Entries[i + 1].Namespace = ns_global; + Entries[i + 1].ResourceID = -1; + Entries[i + 1].FileName = file->NormalizeFileName(fn); + Entries[i + 1].Method = METHOD_STORED; + if (strstr(fn, ".GRP")) Entries[i + 1].Flags |= RESFF_EMBEDDED; j += flength; - Reader.Seek(104, FileReader::SeekCur); + Reader->Seek(104, FileReader::SeekCur); + file->GenerateHash(); } return true; } @@ -145,8 +129,8 @@ FResourceFile* CheckSSI(const char* filename, FileReader& file, LumpFilterInfo* { if (!skipstring(70)) return nullptr; } - auto ssi = new FSSIFile(filename, file, sp); - if (ssi->Open(version, numfiles, filter)) return ssi; + auto ssi = new FResourceFile(filename, file, sp); + if (OpenSSI(ssi, version, numfiles, filter)) return ssi; file = ssi->Destroy(); } } diff --git a/source/common/filesystem/source/file_wad.cpp b/source/common/filesystem/source/file_wad.cpp index 3451b7da8..c35d4ce59 100644 --- a/source/common/filesystem/source/file_wad.cpp +++ b/source/common/filesystem/source/file_wad.cpp @@ -37,6 +37,8 @@ #include "resourcefile.h" #include "fs_filesystem.h" #include "fs_swap.h" +#include "fs_stringpool.h" +#include "resourcefile.h" namespace FileSys { using namespace byteswap; @@ -56,71 +58,6 @@ struct wadlump_t char Name[8]; }; -//========================================================================== -// -// Wad Lump (with console doom LZSS support) -// -//========================================================================== - -class FWadFileLump : public FResourceLump -{ -public: - bool Compressed; - int Position; - int Namespace; - - int GetNamespace() const override { return Namespace; } - - int GetFileOffset() override { return Position; } - FileReader *GetReader() override - { - if(!Compressed) - { - Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet); - return Owner->GetContainerReader(); - } - return NULL; - } - int FillCache() override - { - if(!Compressed) - { - const char * buffer = Owner->GetContainerReader()->GetBuffer(); - - if (buffer != NULL) - { - // This is an in-memory file so the cache can point directly to the file's data. - Cache = const_cast(buffer) + Position; - RefCount = -1; - return -1; - } - } - - Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet); - Cache = new char[LumpSize]; - - if(Compressed) - { - FileReader lzss; - if (lzss.OpenDecompressor(*Owner->GetContainerReader(), LumpSize, METHOD_LZSS, false, true)) - { - lzss.Read(Cache, LumpSize); - } - } - else - { - auto read = Owner->GetContainerReader()->Read(Cache, LumpSize); - if (read != LumpSize) - { - throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize); - } - } - - RefCount = 1; - return 1; - } -}; - //========================================================================== // // Wad file @@ -129,15 +66,12 @@ public: class FWadFile : public FResourceFile { - TArray Lumps; - bool IsMarker(int lump, const char *marker); void SetNamespace(const char *startmarker, const char *endmarker, namespace_t space, FileSystemMessageFunc Printf, bool flathack=false); void SkinHack (FileSystemMessageFunc Printf); public: FWadFile(const char * filename, FileReader &file, StringPool* sp); - FResourceLump *GetLump(int lump) { return &Lumps[lump]; } bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf); }; @@ -188,56 +122,53 @@ bool FWadFile::Open(LumpFilterInfo*, FileSystemMessageFunc Printf) } } - TArray fileinfo(NumLumps, true); - Reader.Seek (InfoTableOfs, FileReader::SeekSet); - Reader.Read (fileinfo.Data(), NumLumps * sizeof(wadlump_t)); + Reader.Seek(InfoTableOfs, FileReader::SeekSet); + auto fd = Reader.Read(NumLumps * sizeof(wadlump_t)); + auto fileinfo = (const wadlump_t*)fd.data(); - Lumps.Resize(NumLumps); + AllocateEntries(NumLumps); for(uint32_t i = 0; i < NumLumps; i++) { + // WAD only supports ASCII. It is also the only format which can use valid backslashes in its names. char n[9]; - for(int j = 0; j < 8; j++) n[j] = toupper(fileinfo[i].Name[j]); - n[8] = 0; - // This needs to be done differently. We cannot simply assume that all lumps where the first character's high bit is set are compressed without verification. - // This requires explicit toggling for precisely the files that need it. -#if 0 - Lumps[i].Compressed = !(gameinfo.flags & GI_SHAREWARE) && (n[0] & 0x80) == 0x80; - n[0] &= ~0x80; -#else - Lumps[i].Compressed = false; -#endif - Lumps[i].LumpNameSetup(n, stringpool); - - Lumps[i].Owner = this; - Lumps[i].Position = isBigEndian ? BigLong(fileinfo[i].FilePos) : LittleLong(fileinfo[i].FilePos); - Lumps[i].LumpSize = isBigEndian ? BigLong(fileinfo[i].Size) : LittleLong(fileinfo[i].Size); - Lumps[i].Namespace = ns_global; - Lumps[i].Flags = Lumps[i].Compressed ? LUMPF_COMPRESSED | LUMPF_SHORTNAME : LUMPF_SHORTNAME; - - // Check if the lump is within the WAD file and print a warning if not. - if (Lumps[i].Position + Lumps[i].LumpSize > wadSize || Lumps[i].Position < 0 || Lumps[i].LumpSize < 0) + int ishigh = 0; + for (int j = 0; j < 8; j++) { - if (Lumps[i].LumpSize != 0) - { - Printf(FSMessageLevel::Warning, "%s: Lump %s contains invalid positioning info and will be ignored\n", FileName, Lumps[i].getName()); - Lumps[i].clearName(); - } - Lumps[i].LumpSize = Lumps[i].Position = 0; + if (fileinfo[i].Name[j] & 0x80) ishigh |= 1 << j; + n[j] = tolower(fileinfo[i].Name[j]); } + n[8] = 0; + if (ishigh == 1) n[0] &= 0x7f; + else if (ishigh > 1) + { + // This may not end up printing something proper because we do not know what encoding might have been used. + Printf(FSMessageLevel::Warning, "%s: Lump name %.8s contains invalid characters\n", FileName, fileinfo[i].Name); + } + + Entries[i].FileName = nullptr; + Entries[i].Position = isBigEndian ? BigLong(fileinfo[i].FilePos) : LittleLong(fileinfo[i].FilePos); + Entries[i].Length = isBigEndian ? BigLong(fileinfo[i].Size) : LittleLong(fileinfo[i].Size); + Entries[i].Namespace = ns_global; + Entries[i].Flags = ishigh? RESFF_SHORTNAME | RESFF_COMPRESSED : RESFF_SHORTNAME; + Entries[i].Method = ishigh == 1? METHOD_LZSS : METHOD_STORED; + Entries[i].FileName = stringpool->Strdup(n); + // This doesn't set up the namespace yet. } + GenerateHash(); // Do this before the lump processing below. - SetNamespace("S_START", "S_END", ns_sprites, Printf); - SetNamespace("F_START", "F_END", ns_flats, Printf, true); - SetNamespace("C_START", "C_END", ns_colormaps, Printf); - SetNamespace("A_START", "A_END", ns_acslibrary, Printf); - SetNamespace("TX_START", "TX_END", ns_newtextures, Printf); - SetNamespace("V_START", "V_END", ns_strifevoices, Printf); - SetNamespace("HI_START", "HI_END", ns_hires, Printf); - SetNamespace("VX_START", "VX_END", ns_voxels, Printf); + SetNamespace("s_start", "s_end", ns_sprites, Printf); + SetNamespace("f_start", "f_end", ns_flats, Printf, true); + SetNamespace("c_start", "c_end", ns_colormaps, Printf); + SetNamespace("a_start", "a_end", ns_acslibrary, Printf); + SetNamespace("tx_start", "tx_end", ns_newtextures, Printf); + SetNamespace("v_start", "v_end", ns_strifevoices, Printf); + SetNamespace("hi_start", "hi_end", ns_hires, Printf); + SetNamespace("vx_start", "vx_end", ns_voxels, Printf); SkinHack(Printf); + return true; } @@ -251,10 +182,10 @@ bool FWadFile::Open(LumpFilterInfo*, FileSystemMessageFunc Printf) inline bool FWadFile::IsMarker(int lump, const char *marker) { - if (Lumps[lump].getName()[0] == marker[0]) + if (Entries[lump].FileName[0] == marker[0]) { - return (!strcmp(Lumps[lump].getName(), marker) || - (marker[1] == '_' && !strcmp(Lumps[lump].getName() +1, marker))); + return (!strcmp(Entries[lump].FileName, marker) || + (marker[1] == '_' && !strcmp(Entries[lump].FileName +1, marker))); } else return false; } @@ -283,20 +214,20 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name bool warned = false; int numstartmarkers = 0, numendmarkers = 0; unsigned int i; - TArray markers; + std::vector markers; for(i = 0; i < NumLumps; i++) { if (IsMarker(i, startmarker)) { Marker m = { 0, i }; - markers.Push(m); + markers.push_back(m); numstartmarkers++; } else if (IsMarker(i, endmarker)) { Marker m = { 1, i }; - markers.Push(m); + markers.push_back(m); numendmarkers++; } } @@ -312,15 +243,15 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name { // We have found no F_START but one or more F_END markers. // mark all lumps before the last F_END marker as potential flats. - unsigned int end = markers[markers.Size()-1].index; + unsigned int end = markers[markers.size()-1].index; for(unsigned int ii = 0; ii < end; ii++) { - if (Lumps[ii].LumpSize == 4096) + if (Entries[ii].Length == 4096) { // We can't add this to the flats namespace but // it needs to be flagged for the texture manager. - Printf(FSMessageLevel::DebugNotify, "%s: Marking %s as potential flat\n", FileName, Lumps[ii].getName()); - Lumps[ii].Flags |= LUMPF_MAYBEFLAT; + Printf(FSMessageLevel::DebugNotify, "%s: Marking %s as potential flat\n", FileName, Entries[ii].FileName); + Entries[ii].Flags |= RESFF_MAYBEFLAT; } } } @@ -328,7 +259,7 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name } i = 0; - while (i < markers.Size()) + while (i < markers.size()) { int start, end; if (markers[i].markertype != 0) @@ -340,21 +271,21 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name start = i++; // skip over subsequent x_START markers - while (i < markers.Size() && markers[i].markertype == 0) + while (i < markers.size() && markers[i].markertype == 0) { Printf(FSMessageLevel::Warning, "%s: duplicate %s marker found.\n", FileName, startmarker); i++; continue; } // same for x_END markers - while (i < markers.Size()-1 && (markers[i].markertype == 1 && markers[i+1].markertype == 1)) + while (i < markers.size()-1 && (markers[i].markertype == 1 && markers[i+1].markertype == 1)) { Printf(FSMessageLevel::Warning, "%s: duplicate %s marker found.\n", FileName, endmarker); i++; continue; } // We found a starting marker but no end marker. Ignore this block. - if (i >= markers.Size()) + if (i >= markers.size()) { Printf(FSMessageLevel::Warning, "%s: %s marker without corresponding %s found.\n", FileName, startmarker, endmarker); end = NumLumps; @@ -368,7 +299,7 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name Printf(FSMessageLevel::DebugNotify, "%s: Found %s block at (%d-%d)\n", FileName, startmarker, markers[start].index, end); for(int j = markers[start].index + 1; j < end; j++) { - if (Lumps[j].Namespace != ns_global) + if (Entries[j].Namespace != ns_global) { if (!warned) { @@ -376,17 +307,17 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name } warned = true; } - else if (space == ns_sprites && Lumps[j].LumpSize < 8) + else if (space == ns_sprites && Entries[j].Length < 8) { // sf 26/10/99: // ignore sprite lumps smaller than 8 bytes (the smallest possible) // in size -- this was used by some dmadds wads // as an 'empty' graphics resource - Printf(FSMessageLevel::DebugWarn, "%s: Skipped empty sprite %s (lump %d)\n", FileName, Lumps[j].getName(), j); + Printf(FSMessageLevel::DebugWarn, "%s: Skipped empty sprite %s (lump %d)\n", FileName, Entries[j].FileName, j); } else { - Lumps[j].Namespace = space; + Entries[j].Namespace = space; } } } @@ -416,11 +347,11 @@ void FWadFile::SkinHack (FileSystemMessageFunc Printf) for (i = 0; i < NumLumps; i++) { - FResourceLump *lump = &Lumps[i]; + auto lump = &Entries[i]; - if (!strnicmp(lump->getName(), "S_SKIN", 6)) + if (!strnicmp(lump->FileName, "S_SKIN", 6)) { // Wad has at least one skin. - lump->LumpNameSetup("S_SKIN", nullptr); + lump->FileName = "S_SKIN"; if (!skinned) { skinned = true; @@ -428,24 +359,24 @@ void FWadFile::SkinHack (FileSystemMessageFunc Printf) for (j = 0; j < NumLumps; j++) { - Lumps[j].Namespace = namespc; + Entries[j].Namespace = namespc; } namespc++; } } // needless to say, this check is entirely useless these days as map names can be more diverse.. - if ((lump->getName()[0] == 'M' && - lump->getName()[1] == 'A' && - lump->getName()[2] == 'P' && - lump->getName()[3] >= '0' && lump->getName()[3] <= '9' && - lump->getName()[4] >= '0' && lump->getName()[4] <= '9' && - lump->getName()[5] == '\0') + if ((lump->FileName[0] == 'M' && + lump->FileName[1] == 'A' && + lump->FileName[2] == 'P' && + lump->FileName[3] >= '0' && lump->FileName[3] <= '9' && + lump->FileName[4] >= '0' && lump->FileName[4] <= '9' && + lump->FileName[5] == '\0') || - (lump->getName()[0] == 'E' && - lump->getName()[1] >= '0' && lump->getName()[1] <= '9' && - lump->getName()[2] == 'M' && - lump->getName()[3] >= '0' && lump->getName()[3] <= '9' && - lump->getName()[4] == '\0')) + (lump->FileName[0] == 'E' && + lump->FileName[1] >= '0' && lump->FileName[1] <= '9' && + lump->FileName[2] == 'M' && + lump->FileName[3] >= '0' && lump->FileName[3] <= '9' && + lump->FileName[4] == '\0')) { hasmap = true; } diff --git a/source/common/filesystem/source/file_whres.cpp b/source/common/filesystem/source/file_whres.cpp index ffa8dd8a5..c0c7be616 100644 --- a/source/common/filesystem/source/file_whres.cpp +++ b/source/common/filesystem/source/file_whres.cpp @@ -34,76 +34,61 @@ ** */ -#include "resourcefile_internal.h" +#include "resourcefile.h" #include "fs_stringpool.h" #include "fs_swap.h" namespace FileSys { using namespace byteswap; -//========================================================================== -// -// WH resource file -// -//========================================================================== - -class FWHResFile : public FUncompressedFile -{ - const char* BaseName; -public: - FWHResFile(const char * filename, FileReader &file, StringPool* sp); - bool Open(LumpFilterInfo* filter); -}; - - -//========================================================================== -// -// -// -//========================================================================== - -FWHResFile::FWHResFile(const char *filename, FileReader &file, StringPool* sp) - : FUncompressedFile(filename, file, sp) -{ - BaseName = stringpool->Strdup(ExtractBaseName(filename, false).c_str()); -} - //========================================================================== // // Open it // //========================================================================== -bool FWHResFile::Open(LumpFilterInfo*) +bool OpenWHRes(FResourceFile* file, LumpFilterInfo*) { uint32_t directory[1024]; - Reader.Seek(-4096, FileReader::SeekEnd); - Reader.Read(directory, 4096); + auto BaseName = ExtractBaseName(file->GetFileName()); + auto Reader = file->GetContainerReader(); + Reader->Seek(-4096, FileReader::SeekEnd); + Reader->Read(directory, 4096); int nl =1024/3; - Lumps.Resize(nl); - + + int k; + for (k = 0; k < nl; k++) + { + uint32_t offset = LittleLong(directory[k * 3]) * 4096; + uint32_t length = LittleLong(directory[k * 3 + 1]); + if (length == 0) + { + break; + } + } + auto Entries = file->AllocateEntries(k); + auto NumLumps = k; int i = 0; - for(int k = 0; k < nl; k++) + for(k = 0; k < NumLumps; k++) { uint32_t offset = LittleLong(directory[k*3]) * 4096; uint32_t length = LittleLong(directory[k*3+1]); - if (length == 0) break; char num[6]; snprintf(num, 6, "/%04d", k); - std::string synthname = BaseName; - synthname += num; - Lumps[i].LumpNameSetup(synthname.c_str(), stringpool); - Lumps[i].Owner = this; - Lumps[i].Position = offset; - Lumps[i].LumpSize = length; + std::string synthname = BaseName + num; + + Entries[i].Position = offset; + Entries[i].Length = length; + Entries[i].Flags = RESFF_FULLPATH; + Entries[i].Namespace = ns_global; + Entries[i].ResourceID = -1; + Entries[i].Method = METHOD_STORED; + Entries[i].FileName = file->NormalizeFileName(synthname.c_str()); i++; } - NumLumps = i; - Lumps.Clamp(NumLumps); - Lumps.ShrinkToFit(); return true; } @@ -134,8 +119,8 @@ FResourceFile *CheckWHRes(const char *filename, FileReader &file, LumpFilterInfo if (offset != checkpos || length == 0 || offset + length >= (size_t)size - 4096 ) return nullptr; checkpos += (length+4095) / 4096; } - auto rf = new FWHResFile(filename, file, sp); - if (rf->Open(filter)) return rf; + auto rf = new FResourceFile(filename, file, sp); + if (OpenWHRes(rf, filter)) return rf; file = rf->Destroy(); } return NULL; diff --git a/source/common/filesystem/source/file_zip.cpp b/source/common/filesystem/source/file_zip.cpp index dfeb41ea8..3356fc1fa 100644 --- a/source/common/filesystem/source/file_zip.cpp +++ b/source/common/filesystem/source/file_zip.cpp @@ -3,7 +3,7 @@ ** **--------------------------------------------------------------------------- ** Copyright 1998-2009 Randy Heit -** Copyright 2005-2009 Christoph Oelckers +** Copyright 2005-2023 Christoph Oelckers ** All rights reserved. ** ** Redistribution and use in source and binary forms, with or without @@ -35,78 +35,18 @@ #include #include -#include "file_zip.h" #include "w_zip.h" #include "ancientzip.h" +#include "resourcefile.h" #include "fs_findfile.h" #include "fs_swap.h" +#include "fs_stringpool.h" namespace FileSys { using namespace byteswap; #define BUFREADCOMMENT (0x400) -//========================================================================== -// -// Decompression subroutine -// -//========================================================================== - -static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdiff_t LumpSize, ptrdiff_t CompressedSize, int GPFlags, bool exceptions) -{ - switch (Method) - { - case METHOD_STORED: - { - Reader.Read(Cache, LumpSize); - break; - } - - case METHOD_DEFLATE: - case METHOD_BZIP2: - case METHOD_LZMA: - case METHOD_XZ: - { - FileReader frz; - if (frz.OpenDecompressor(Reader, LumpSize, Method, false, exceptions)) - { - frz.Read(Cache, LumpSize); - } - break; - } - - // Fixme: These should also use a stream - case METHOD_IMPLODE: - { - FZipExploder exploder; - if (exploder.Explode((unsigned char*)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize, GPFlags) == -1) - { - // decompression failed so zero the cache. - memset(Cache, 0, LumpSize); - } - break; - } - - case METHOD_SHRINK: - { - ShrinkLoop((unsigned char *)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize); - break; - } - - default: - assert(0); - return false; - } - return true; -} - -bool FCompressedBuffer::Decompress(char *destbuffer) -{ - FileReader mr; - mr.OpenMemory(mBuffer, mCompressedSize); - return UncompressZipLump(destbuffer, mr, mMethod, mSize, mCompressedSize, mZipFlags, false); -} - //----------------------------------------------------------------------- // // Finds the central directory end record in the end of the file. @@ -166,10 +106,25 @@ static uint32_t Zip_FindCentralDir(FileReader &fin, bool* zip64) // //========================================================================== +class FZipFile : public FResourceFile +{ + void SetEntryAddress(uint32_t entry) override; + +public: + FZipFile(const char* filename, FileReader& file, StringPool* sp); + bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf); + FCompressedBuffer GetRawData(uint32_t entry) override; +}; + +//========================================================================== +// +// Zip file +// +//========================================================================== + FZipFile::FZipFile(const char * filename, FileReader &file, StringPool* sp) : FResourceFile(filename, file, sp) { - Lumps = NULL; } bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) @@ -178,8 +133,6 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) uint32_t centraldir = Zip_FindCentralDir(Reader, &zip64); int skipped = 0; - Lumps = NULL; - if (centraldir == 0) { Printf(FSMessageLevel::Error, "%s: ZIP file corrupt!\n", FileName); @@ -225,15 +178,12 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) dirsize = info.DirectorySize; DirectoryOffset = info.DirectoryOffset; } - Lumps = new FZipLump[NumLumps]; - // Load the entire central directory. Too bad that this contains variable length entries... void *directory = malloc(dirsize); Reader.Seek(DirectoryOffset, FileReader::SeekSet); Reader.Read(directory, dirsize); char *dirptr = (char*)directory; - FZipLump *lump_p = Lumps; std::string name0, name1; bool foundspeciallump = false; @@ -259,58 +209,11 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) Printf(FSMessageLevel::Error, "%s: Central directory corrupted.", FileName); return false; } - - for (auto& c : name) c = tolower(c); - - auto vv = name.find("__macosx"); - if (name.find("filter/") == 0) - continue; // 'filter' is a reserved name of the file system. - if (name.find("__macosx") == 0) - continue; // skip Apple garbage. At this stage only the root folder matters. - if (name.find(".bat") != std::string::npos || name.find(".exe") != std::string::npos) - continue; // also ignore executables for this. - if (!foundprefix) - { - // check for special names, if one of these gets found this must be treated as a normal zip. - bool isspecial = name.find("/") == std::string::npos || - (filter && std::find(filter->reservedFolders.begin(), filter->reservedFolders.end(), name) != filter->reservedFolders.end()); - if (isspecial) break; - name0 = std::string(name, 0, name.rfind("/")+1); - name1 = std::string(name, 0, name.find("/") + 1); - foundprefix = true; - } - - if (name.find(name0) != 0) - { - if (!name1.empty()) - { - name0 = name1; - if (name.find(name0) != 0) - { - name0 = ""; - } - } - if (name0.empty()) - break; - } - if (!foundspeciallump && filter) - { - // at least one of the more common definition lumps must be present. - for (auto &p : filter->requiredPrefixes) - { - if (name.find(name0 + p) == 0 || name.rfind(p) == size_t(name.length() - p.length())) - { - foundspeciallump = true; - break; - } - } - } } - // If it ran through the list without finding anything it should not attempt any path remapping. - if (!foundspeciallump) name0 = ""; dirptr = (char*)directory; - lump_p = Lumps; + AllocateEntries(NumLumps); + auto Entry = Entries; for (uint32_t i = 0; i < NumLumps; i++) { FZipCentralDirectoryInfo *zip_fh = (FZipCentralDirectoryInfo *)dirptr; @@ -329,13 +232,6 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) return false; } - if (name.find("__macosx") == 0 || name.find("__MACOSX") == 0) - { - skipped++; - continue; // Weed out Apple's resource fork garbage right here because it interferes with safe operation. - } - if (!name0.empty()) name = std::string(name, name0.length()); - // skip Directories if (name.empty() || (name.back() == '/' && LittleLong(zip_fh->UncompressedSize32) == 0)) { @@ -366,9 +262,6 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) continue; } - FixPathSeparator(&name.front()); - for (auto& c : name) c = tolower(c); - uint32_t UncompressedSize =LittleLong(zip_fh->UncompressedSize32); uint32_t CompressedSize = LittleLong(zip_fh->CompressedSize32); uint64_t LocalHeaderOffset = LittleLong(zip_fh->LocalHeaderOffset32); @@ -399,54 +292,59 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) } } - lump_p->LumpNameSetup(name.c_str(), stringpool); - lump_p->LumpSize = UncompressedSize; - lump_p->Owner = this; + Entry->FileName = NormalizeFileName(name.c_str()); + Entry->Length = UncompressedSize; // The start of the Reader will be determined the first time it is accessed. - lump_p->Flags = LUMPF_FULLPATH; - lump_p->NeedFileStart = true; - lump_p->Method = uint8_t(zip_fh->Method); - if (lump_p->Method != METHOD_STORED) lump_p->Flags |= LUMPF_COMPRESSED; - lump_p->GPFlags = zip_fh->Flags; - lump_p->CRC32 = zip_fh->CRC32; - lump_p->CompressedSize = CompressedSize; - lump_p->Position = LocalHeaderOffset; - lump_p->CheckEmbedded(filter); + Entry->Flags = RESFF_FULLPATH | RESFF_NEEDFILESTART; + Entry->Method = uint8_t(zip_fh->Method); + if (Entry->Method != METHOD_STORED) Entry->Flags |= RESFF_COMPRESSED; + if (Entry->Method == METHOD_IMPLODE) + { + // for Implode merge the flags into the compression method to make handling in the file system easier and save one variable. + if ((zip_fh->Flags & 6) == 2) Entry->Method = METHOD_IMPLODE_2; + else if ((zip_fh->Flags & 6) == 4) Entry->Method = METHOD_IMPLODE_4; + else if ((zip_fh->Flags & 6) == 6) Entry->Method = METHOD_IMPLODE_6; + else Entry->Method = METHOD_IMPLODE_0; + } + Entry->CRC32 = zip_fh->CRC32; + Entry->CompressedSize = CompressedSize; + Entry->Position = LocalHeaderOffset; + + Entry++; - lump_p++; } // Resize the lump record array to its actual size NumLumps -= skipped; free(directory); GenerateHash(); - PostProcessArchive(&Lumps[0], sizeof(FZipLump), filter); + PostProcessArchive(filter); return true; } //========================================================================== // -// Zip file +// // //========================================================================== -FZipFile::~FZipFile() +FCompressedBuffer FZipFile::GetRawData(uint32_t entry) { - if (Lumps != NULL) delete [] Lumps; -} + FCompressedBuffer cbuf; -//========================================================================== -// -// -// -//========================================================================== - -FCompressedBuffer FZipLump::GetRawData() -{ - FCompressedBuffer cbuf = { (unsigned)LumpSize, (unsigned)CompressedSize, Method, GPFlags, CRC32, new char[CompressedSize] }; - if (NeedFileStart) SetLumpAddress(); - Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet); - Owner->GetContainerReader()->Read(cbuf.mBuffer, CompressedSize); + if (entry >= NumLumps >> Entries[entry].Length == 0) + { + cbuf = { 0, 0, METHOD_STORED, 0, 0, nullptr }; + } + else + { + auto& e = Entries[entry]; + cbuf = { e.Length, e.CompressedSize, e.Method, e.CRC32, new char[e.CompressedSize] }; + if (e.Flags & RESFF_NEEDFILESTART) SetEntryAddress(entry); + Reader.Seek(e.Position, FileReader::SeekSet); + Reader.Read(cbuf.mBuffer, e.CompressedSize); + } + return cbuf; } @@ -456,7 +354,7 @@ FCompressedBuffer FZipLump::GetRawData() // //========================================================================== -void FZipLump::SetLumpAddress() +void FZipFile::SetEntryAddress(uint32_t entry) { // This file is inside a zip and has not been opened before. // Position points to the start of the local file header, which we must @@ -464,69 +362,11 @@ void FZipLump::SetLumpAddress() FZipLocalFileHeader localHeader; int skiplen; - Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet); - Owner->GetContainerReader()->Read(&localHeader, sizeof(localHeader)); + Reader.Seek(Entries[entry].Position, FileReader::SeekSet); + Reader.Read(&localHeader, sizeof(localHeader)); skiplen = LittleShort(localHeader.NameLength) + LittleShort(localHeader.ExtraLength); - Position += sizeof(localHeader) + skiplen; - NeedFileStart = false; -} - -//========================================================================== -// -// Get reader (only returns non-NULL if not encrypted) -// -//========================================================================== - -FileReader *FZipLump::GetReader() -{ - // Don't return the reader if this lump is encrypted - // In that case always force caching of the lump - if (Method == METHOD_STORED) - { - if (NeedFileStart) SetLumpAddress(); - Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet); - return Owner->GetContainerReader(); - } - else return NULL; -} - -//========================================================================== -// -// Fills the lump cache and performs decompression -// -//========================================================================== - -int FZipLump::FillCache() -{ - if (NeedFileStart) SetLumpAddress(); - const char *buffer; - - if (Method == METHOD_STORED && (buffer = Owner->GetContainerReader()->GetBuffer()) != NULL) - { - // This is an in-memory file so the cache can point directly to the file's data. - Cache = const_cast(buffer) + Position; - RefCount = -1; - return -1; - } - - Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet); - Cache = new char[LumpSize]; - UncompressZipLump(Cache, *Owner->GetContainerReader(), Method, LumpSize, CompressedSize, GPFlags, true); - RefCount = 1; - return 1; -} - -//========================================================================== -// -// -// -//========================================================================== - -int FZipLump::GetFileOffset() -{ - if (Method != METHOD_STORED) return -1; - if (NeedFileStart) SetLumpAddress(); - return (int)Position; + Entries[entry].Position += sizeof(localHeader) + skiplen; + Entries[entry].Flags &= ~RESFF_NEEDFILESTART; } //========================================================================== @@ -555,171 +395,4 @@ FResourceFile *CheckZip(const char *filename, FileReader &file, LumpFilterInfo* } - -//========================================================================== -// -// time_to_dos -// -// Converts time from struct tm to the DOS format used by zip files. -// -//========================================================================== - -static std::pair time_to_dos(struct tm *time) -{ - std::pair val; - if (time == NULL || time->tm_year < 80) - { - val.first = val.second = 0; - } - else - { - val.first = (time->tm_year - 80) * 512 + (time->tm_mon + 1) * 32 + time->tm_mday; - val.second= time->tm_hour * 2048 + time->tm_min * 32 + time->tm_sec / 2; - } - return val; -} - -//========================================================================== -// -// append_to_zip -// -// Write a given file to the zipFile. -// -// zipfile: zip object to be written to -// -// returns: position = success, -1 = error -// -//========================================================================== - -static int AppendToZip(FileWriter *zip_file, const FCompressedBuffer &content, std::pair &dostime) -{ - FZipLocalFileHeader local; - int position; - - local.Magic = ZIP_LOCALFILE; - local.VersionToExtract[0] = 20; - local.VersionToExtract[1] = 0; - local.Flags = content.mMethod == METHOD_DEFLATE ? LittleShort((uint16_t)2) : LittleShort((uint16_t)content.mZipFlags); - local.Method = LittleShort((uint16_t)content.mMethod); - local.ModDate = LittleShort(dostime.first); - local.ModTime = LittleShort(dostime.second); - local.CRC32 = content.mCRC32; - local.UncompressedSize = LittleLong((unsigned)content.mSize); - local.CompressedSize = LittleLong((unsigned)content.mCompressedSize); - local.NameLength = LittleShort((unsigned short)strlen(content.filename)); - local.ExtraLength = 0; - - // Fill in local directory header. - - position = (int)zip_file->Tell(); - - // Write out the header, file name, and file data. - if (zip_file->Write(&local, sizeof(local)) != sizeof(local) || - zip_file->Write(content.filename, strlen(content.filename)) != strlen(content.filename) || - zip_file->Write(content.mBuffer, content.mCompressedSize) != content.mCompressedSize) - { - return -1; - } - return position; -} - - -//========================================================================== -// -// write_central_dir -// -// Writes the central directory entry for a file. -// -//========================================================================== - -int AppendCentralDirectory(FileWriter *zip_file, const FCompressedBuffer &content, std::pair &dostime, int position) -{ - FZipCentralDirectoryInfo dir; - - dir.Magic = ZIP_CENTRALFILE; - dir.VersionMadeBy[0] = 20; - dir.VersionMadeBy[1] = 0; - dir.VersionToExtract[0] = 20; - dir.VersionToExtract[1] = 0; - dir.Flags = content.mMethod == METHOD_DEFLATE ? LittleShort((uint16_t)2) : LittleShort((uint16_t)content.mZipFlags); - dir.Method = LittleShort((uint16_t)content.mMethod); - dir.ModTime = LittleShort(dostime.first); - dir.ModDate = LittleShort(dostime.second); - dir.CRC32 = content.mCRC32; - dir.CompressedSize32 = LittleLong((unsigned)content.mCompressedSize); - dir.UncompressedSize32 = LittleLong((unsigned)content.mSize); - dir.NameLength = LittleShort((unsigned short)strlen(content.filename)); - dir.ExtraLength = 0; - dir.CommentLength = 0; - dir.StartingDiskNumber = 0; - dir.InternalAttributes = 0; - dir.ExternalAttributes = 0; - dir.LocalHeaderOffset32 = LittleLong((unsigned)position); - - if (zip_file->Write(&dir, sizeof(dir)) != sizeof(dir) || - zip_file->Write(content.filename, strlen(content.filename)) != strlen(content.filename)) - { - return -1; - } - return 0; -} - -bool WriteZip(const char* filename, const FCompressedBuffer* content, size_t contentcount) -{ - // try to determine local time - struct tm *ltime; - time_t ttime; - ttime = time(nullptr); - ltime = localtime(&ttime); - auto dostime = time_to_dos(ltime); - - TArray positions; - - auto f = FileWriter::Open(filename); - if (f != nullptr) - { - for (size_t i = 0; i < contentcount; i++) - { - int pos = AppendToZip(f, content[i], dostime); - if (pos == -1) - { - delete f; - remove(filename); - return false; - } - positions.Push(pos); - } - - int dirofs = (int)f->Tell(); - for (size_t i = 0; i < contentcount; i++) - { - if (AppendCentralDirectory(f, content[i], dostime, positions[i]) < 0) - { - delete f; - remove(filename); - return false; - } - } - - // Write the directory terminator. - FZipEndOfCentralDirectory dirend; - dirend.Magic = ZIP_ENDOFDIR; - dirend.DiskNumber = 0; - dirend.FirstDisk = 0; - dirend.NumEntriesOnAllDisks = dirend.NumEntries = LittleShort((uint16_t)contentcount); - dirend.DirectoryOffset = LittleLong((unsigned)dirofs); - dirend.DirectorySize = LittleLong((uint32_t)(f->Tell() - dirofs)); - dirend.ZipCommentLength = 0; - if (f->Write(&dirend, sizeof(dirend)) != sizeof(dirend)) - { - delete f; - remove(filename); - return false; - } - delete f; - return true; - } - return false; -} - } diff --git a/source/common/filesystem/source/file_zip.h b/source/common/filesystem/source/file_zip.h deleted file mode 100644 index 8313c9eb2..000000000 --- a/source/common/filesystem/source/file_zip.h +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef __FILE_ZIP_H -#define __FILE_ZIP_H - -#include "resourcefile.h" - -namespace FileSys { -//========================================================================== -// -// Zip Lump -// -//========================================================================== - -struct FZipLump : public FResourceLump -{ - uint16_t GPFlags; - uint8_t Method; - bool NeedFileStart; - int CompressedSize; - int64_t Position; - unsigned CRC32; - - virtual FileReader *GetReader(); - virtual int FillCache() override; - -private: - void SetLumpAddress(); - virtual int GetFileOffset(); - FCompressedBuffer GetRawData(); -}; - - -//========================================================================== -// -// Zip file -// -//========================================================================== - -class FZipFile : public FResourceFile -{ - FZipLump *Lumps; - -public: - FZipFile(const char * filename, FileReader &file, StringPool* sp); - virtual ~FZipFile(); - bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf); - virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; } -}; - -} - -#endif \ No newline at end of file diff --git a/source/common/filesystem/source/files.cpp b/source/common/filesystem/source/files.cpp index 7bf013060..ac0581119 100644 --- a/source/common/filesystem/source/files.cpp +++ b/source/common/filesystem/source/files.cpp @@ -35,6 +35,9 @@ #include #include +#include +#include +#include #include "files_internal.h" namespace FileSys { @@ -326,15 +329,15 @@ char *MemoryReader::Gets(char *strbuf, ptrdiff_t len) int BufferingReader::FillBuffer(ptrdiff_t newpos) { if (newpos > Length) newpos = Length; - if (newpos < bufferpos) return 0; - auto read = baseReader->Read(&buf[bufferpos], newpos - bufferpos); + if (newpos <= bufferpos) return 0; + auto read = baseReader->Read(&buf.writable()[bufferpos], newpos - bufferpos); bufferpos += read; if (bufferpos == Length) { // we have read the entire file, so delete our data provider. baseReader.reset(); } - return read == newpos - bufferpos ? 0 : -1; + return newpos == bufferpos ? 0 : -1; } ptrdiff_t BufferingReader::Seek(ptrdiff_t offset, int origin) @@ -392,45 +395,40 @@ bool FileReader::OpenMemory(const void *mem, FileReader::Size length) return true; } -bool FileReader::OpenMemoryArray(const void *mem, FileReader::Size length) +bool FileReader::OpenMemoryArray(FileData& data) { Close(); - mReader = new MemoryArrayReader>((const char *)mem, length); + if (data.size() > 0) mReader = new MemoryArrayReader(data); return true; } -bool FileReader::OpenMemoryArray(std::vector& data) +FileData FileReader::Read(size_t len) { - Close(); - if (data.size() > 0) mReader = new MemoryArrayReader>(data); - return true; -} - -bool FileReader::OpenMemoryArray(ResourceData& data) -{ - Close(); - if (data.size() > 0) mReader = new MemoryArrayReader(data); - return true; -} - -bool FileReader::OpenMemoryArray(std::function&)> getter) -{ - auto reader = new MemoryArrayReader>(nullptr, 0); - if (getter(reader->GetArray())) + FileData buffer; + if (len > 0) { - Close(); - reader->UpdateBuffer(); - mReader = reader; - return true; - } - else - { - // This will keep the old buffer, if one existed - delete reader; - return false; + Size length = mReader->Read(buffer.allocate(len), len); + if ((size_t)length < len) buffer.allocate(length); } + return buffer; } +FileData FileReader::ReadPadded(size_t padding) +{ + auto len = GetLength(); + FileData buffer; + + if (len > 0) + { + auto p = (char*)buffer.allocate(len + padding); + Size length = mReader->Read(p, len); + if (length < len) buffer.clear(); + else memset(p + len, 0, padding); + } + return buffer; +} + + //========================================================================== // diff --git a/source/common/filesystem/source/files_decompress.cpp b/source/common/filesystem/source/files_decompress.cpp index c5552018f..95c6c7f9b 100644 --- a/source/common/filesystem/source/files_decompress.cpp +++ b/source/common/filesystem/source/files_decompress.cpp @@ -45,6 +45,9 @@ #include #include "fs_files.h" +#include "files_internal.h" +#include "ancientzip.h" +#include "fs_decompress.h" namespace FileSys { using namespace byteswap; @@ -53,6 +56,7 @@ namespace FileSys { class DecompressorBase : public FileReaderInterface { bool exceptions = false; + public: // These do not work but need to be defined to satisfy the FileReaderInterface. // They will just error out when called. @@ -64,6 +68,10 @@ public: void EnableExceptions(bool on) { exceptions = on; } protected: + DecompressorBase() + { + //seekable = false; + } FileReader* File = nullptr; FileReader OwnedFile; }; @@ -174,7 +182,7 @@ public: inflateEnd (&Stream); } - ptrdiff_t Read (void *buffer, ptrdiff_t len) override + ptrdiff_t Read (void *buffer, ptrdiff_t olen) override { int err = 0; @@ -183,6 +191,7 @@ public: DecompressionError("File not open"); return 0; } + auto len = olen; if (len == 0) return 0; while (len > 0) @@ -215,7 +224,7 @@ public: return 0; } - return len - Stream.avail_out; + return olen - Stream.avail_out; } void FillBuffer () @@ -841,19 +850,22 @@ public: }; -bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions) +bool OpenDecompressor(FileReader& self, FileReader &parent, FileReader::Size length, int method, int flags) { + FileReaderInterface* fr = nullptr; DecompressorBase* dec = nullptr; try { FileReader* p = &parent; - switch (method & ~METHOD_TRANSFEROWNER) + bool exceptions = !!(flags & DCF_EXCEPTIONS); + + switch (method) { case METHOD_DEFLATE: case METHOD_ZLIB: { auto idec = new DecompressorZ; - dec = idec; + fr = dec = idec; idec->EnableExceptions(exceptions); if (!idec->Open(p, method == METHOD_DEFLATE)) { @@ -865,7 +877,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b case METHOD_BZIP2: { auto idec = new DecompressorBZ2; - dec = idec; + fr = dec = idec; idec->EnableExceptions(exceptions); if (!idec->Open(p)) { @@ -877,7 +889,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b case METHOD_LZMA: { auto idec = new DecompressorLZMA; - dec = idec; + fr = dec = idec; idec->EnableExceptions(exceptions); if (!idec->Open(p, length)) { @@ -889,7 +901,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b case METHOD_XZ: { auto idec = new DecompressorXZ; - dec = idec; + fr = dec = idec; idec->EnableExceptions(exceptions); if (!idec->Open(p, length)) { @@ -901,7 +913,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b case METHOD_LZSS: { auto idec = new DecompressorLZSS; - dec = idec; + fr = dec = idec; idec->EnableExceptions(exceptions); if (!idec->Open(p)) { @@ -911,34 +923,103 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b break; } - // todo: METHOD_IMPLODE, METHOD_SHRINK + // The decoders for these legacy formats can only handle the full data in one go so we have to perform the entire decompression here. + case METHOD_IMPLODE_0: + case METHOD_IMPLODE_2: + case METHOD_IMPLODE_4: + case METHOD_IMPLODE_6: + { + FileData buffer(nullptr, length); + FZipExploder exploder; + if (exploder.Explode(buffer.writable(), length, *p, p->GetLength(), method - METHOD_IMPLODE_MIN) == -1) + { + if (exceptions) + { + throw FileSystemException("DecompressImplode failed"); + } + return false; + } + fr = new MemoryArrayReader(buffer); + flags &= ~(DCF_SEEKABLE | DCF_CACHED); + break; + } + + case METHOD_SHRINK: + { + FileData buffer(nullptr, length); + ShrinkLoop(buffer.writable(), length, *p, p->GetLength()); // this never fails. + fr = new MemoryArrayReader(buffer); + flags &= ~(DCF_SEEKABLE | DCF_CACHED); + break; + } + + // While this could be made a buffering reader it isn't worth the effort because only stock RFFs are encrypted and they do not contain large files. + case METHOD_RFFCRYPT: + { + FileData buffer = p->Read(length); + auto bufr = buffer.writable(); + FileReader::Size cryptlen = std::min(length, 256); + + for (FileReader::Size i = 0; i < cryptlen; ++i) + { + bufr[i] ^= i >> 1; + } + fr = new MemoryArrayReader(buffer); + flags &= ~(DCF_SEEKABLE | DCF_CACHED); + break; + } + default: return false; } - if (method & METHOD_TRANSFEROWNER) + if (dec) { - dec->SetOwnsReader(); + if (flags & DCF_TRANSFEROWNER) + { + dec->SetOwnsReader(); + } + dec->Length = length; } - - dec->Length = length; - if (!seekable) + if ((flags & DCF_CACHED)) { - Close(); - mReader = dec; - return true; + // read everything into a MemoryArrayReader. + FileData data(nullptr, length); + fr->Read(data.writable(), length); + fr = new MemoryArrayReader(data); } - else + else if ((flags & DCF_SEEKABLE)) { - // todo: create a wrapper. for now this fails - delete dec; - return false; + // create a wrapper that can buffer the content so that seeking is possible + fr = new BufferingReader(fr); } - } + self = FileReader(fr); + return true; + } catch (...) { - if (dec) delete dec; + if (fr) delete fr; throw; } } + +bool FCompressedBuffer::Decompress(char* destbuffer) +{ + if (mMethod == METHOD_STORED) + { + memcpy(destbuffer, mBuffer, mSize); + return true; + } + else + { + FileReader mr; + mr.OpenMemory(mBuffer, mCompressedSize); + FileReader frz; + if (OpenDecompressor(frz, mr, mSize, mMethod)) + { + return frz.Read(destbuffer, mSize) != mSize; + } + } + return false; } +} \ No newline at end of file diff --git a/source/common/filesystem/source/files_internal.h b/source/common/filesystem/source/files_internal.h index aa3595cbf..08f486599 100644 --- a/source/common/filesystem/source/files_internal.h +++ b/source/common/filesystem/source/files_internal.h @@ -31,7 +31,7 @@ public: class BufferingReader : public MemoryReader { - std::vector buf; + FileData buf; std::unique_ptr baseReader; ptrdiff_t bufferpos = 0; @@ -40,6 +40,9 @@ public: BufferingReader(FileReaderInterface* base) : baseReader(base) { + Length = base->Length; + buf.allocate(Length); + bufptr = (const char*)buf.data(); } ptrdiff_t Seek(ptrdiff_t offset, int origin) override; @@ -55,10 +58,9 @@ public: // //========================================================================== -template class MemoryArrayReader : public MemoryReader { - T buf; + FileData buf; public: MemoryArrayReader() @@ -67,24 +69,18 @@ public: Length = 0; } - MemoryArrayReader(const char* buffer, ptrdiff_t length) + MemoryArrayReader(size_t len) { - if (length > 0) - { - buf.resize(length); - memcpy(&buf[0], buffer, length); - } + buf.allocate(len); UpdateBuffer(); } - MemoryArrayReader(T& buffer) + MemoryArrayReader(FileData& buffer) { buf = std::move(buffer); UpdateBuffer(); } - std::vector& GetArray() { return buf; } - void UpdateBuffer() { bufptr = (const char*)buf.data(); @@ -93,7 +89,4 @@ public: } }; -bool OpenMemoryArray(std::vector& data); // take the given array - - } diff --git a/source/common/filesystem/source/filesystem.cpp b/source/common/filesystem/source/filesystem.cpp index 7f36e513a..394735d85 100644 --- a/source/common/filesystem/source/filesystem.cpp +++ b/source/common/filesystem/source/filesystem.cpp @@ -41,7 +41,7 @@ #include #include -#include "resourcefile_internal.h" +#include "resourcefile.h" #include "fs_filesystem.h" #include "fs_findfile.h" #include "md5.hpp" @@ -90,29 +90,37 @@ static void md5Hash(FileReader& reader, uint8_t* digest) struct FileSystem::LumpRecord { - FResourceLump *lump; + FResourceFile *resfile; LumpShortName shortName; const char* LongName; - int rfnum; - int Namespace; + int resindex; + int16_t rfnum; // this is not necessarily the same as resfile's index! + int16_t Namespace; int resourceId; int flags; - void SetFromLump(int filenum, FResourceLump* lmp, StringPool* sp) + void SetFromLump(FResourceFile* file, int fileindex, int filenum, StringPool* sp, const char* name = nullptr) { - lump = lmp; + if (fileindex == 649 && filenum == 0) + { + int a = 0; + } + resfile = file; + resindex = fileindex; rfnum = filenum; flags = 0; - if (lump->Flags & LUMPF_SHORTNAME) + auto lflags = file->GetEntryFlags(fileindex); + if (!name) name = file->getName(fileindex); + if (lflags & RESFF_SHORTNAME) { - UpperCopy(shortName.String, lump->getName()); + UpperCopy(shortName.String, name); shortName.String[8] = 0; LongName = ""; - Namespace = lump->GetNamespace(); + Namespace = file->GetEntryNamespace(fileindex); resourceId = -1; } - else if ((lump->Flags & LUMPF_EMBEDDED) || !lump->getName() || !*lump->getName()) + else if ((lflags & RESFF_EMBEDDED) || !name || !*name) { shortName.qword = 0; LongName = ""; @@ -121,8 +129,8 @@ struct FileSystem::LumpRecord } else { - LongName = lump->getName(); - resourceId = lump->GetIndexNum(); + LongName = name; + resourceId = file->GetEntryResourceID(fileindex); // Map some directories to WAD namespaces. // Note that some of these namespaces don't exist in WADS. @@ -206,11 +214,6 @@ void FileSystem::DeleteAll () Hashes.clear(); NumEntries = 0; - // explicitly delete all manually added lumps. - for (auto &frec : FileInfo) - { - if (frec.rfnum == -1) delete frec.lump; - } FileInfo.clear(); for (int i = (int)Files.size() - 1; i >= 0; --i) { @@ -242,6 +245,8 @@ bool FileSystem::InitMultipleFiles (std::vector& filenames, LumpFil { int numfiles; + // the first call here will designate a main thread which may use shared file readers. All other thewads have to open new file handles. + SetMainThread(); // open all the files, load headers, and count lumps DeleteAll(); numfiles = 0; @@ -295,23 +300,23 @@ bool FileSystem::InitMultipleFiles (std::vector& filenames, LumpFil // //========================================================================== -FResourceFile* CheckLump(const char* filename, FileReader& file, LumpFilterInfo*, FileSystemMessageFunc Printf, StringPool* sp); - int FileSystem::AddFromBuffer(const char* name, char* data, int size, int id, int flags) { FileReader fr; - fr.OpenMemoryArray((uint8_t*)data, size); + FileData blob(data, size); + fr.OpenMemoryArray(blob); - // just wrap this into a single lump resource file (should be done a little better later.) - auto rf = CheckLump(name, fr, nullptr, nullptr, stringpool); - if (rf) - { - Files.push_back(rf); - FResourceLump* lump = rf->GetLump(0); - FileInfo.resize(FileInfo.size() + 1); - FileSystem::LumpRecord* lump_p = &FileInfo.back(); - lump_p->SetFromLump((int)Files.size(), lump, stringpool); - } + // wrap this into a single lump resource file (should be done a little better later.) + auto rf = new FResourceFile(name, fr, stringpool); + auto Entries = rf->AllocateEntries(1); + Entries[0].FileName = rf->NormalizeFileName(ExtractBaseName(name, true).c_str()); + Entries[0].ResourceID = -1; + Entries[0].Length = size; + + Files.push_back(rf); + FileInfo.resize(FileInfo.size() + 1); + FileSystem::LumpRecord* lump_p = &FileInfo.back(); + lump_p->SetFromLump(rf, 0, (int)Files.size() - 1, stringpool); return (int)FileInfo.size() - 1; } @@ -377,25 +382,23 @@ void FileSystem::AddFile (const char *filename, FileReader *filer, LumpFilterInf uint32_t lumpstart = (uint32_t)FileInfo.size(); resfile->SetFirstLump(lumpstart); + Files.push_back(resfile); for (int i = 0; i < resfile->EntryCount(); i++) { - FResourceLump* lump = resfile->GetLump(i); FileInfo.resize(FileInfo.size() + 1); FileSystem::LumpRecord* lump_p = &FileInfo.back(); - lump_p->SetFromLump((int)Files.size(), lump, stringpool); + lump_p->SetFromLump(resfile, i, (int)Files.size() - 1, stringpool); } - Files.push_back(resfile); - for (int i = 0; i < resfile->EntryCount(); i++) { int flags = resfile->GetEntryFlags(i); - if (flags & LUMPF_EMBEDDED) + if (flags & RESFF_EMBEDDED) { std::string path = filename; path += ':'; path += resfile->getName(i); - auto embedded = resfile->GetEntryReader(i, true); + auto embedded = resfile->GetEntryReader(i, READER_NEW, READERFLAG_SEEKABLE); AddFile(path.c_str(), &embedded, filter, Printf, hashfile); } } @@ -425,9 +428,9 @@ void FileSystem::AddFile (const char *filename, FileReader *filer, LumpFilterInf for (int i = 0; i < resfile->EntryCount(); i++) { int flags = resfile->GetEntryFlags(i); - if (!(flags & LUMPF_EMBEDDED)) + if (!(flags & RESFF_EMBEDDED)) { - auto reader = resfile->GetEntryReader(i, true); + auto reader = resfile->GetEntryReader(i, READER_SHARED, 0); md5Hash(filereader, cksum); for (size_t j = 0; j < sizeof(cksum); ++j) @@ -529,8 +532,9 @@ int FileSystem::CheckNumForName (const char *name, int space) const // If we find a lump with this name in the global namespace that does not come // from a Zip return that. WADs don't know these namespaces and single lumps must // work as well. + auto lflags = lump.resfile->GetEntryFlags(lump.resindex); if (space > ns_specialzipdirectory && lump.Namespace == ns_global && - !((lump.lump->Flags ^lump.flags) & LUMPF_FULLPATH)) break; + !((lflags ^lump.flags) & RESFF_FULLPATH)) break; } i = NextLumpIndex[i]; } @@ -770,31 +774,14 @@ int FileSystem::GetResource (int resid, const char *type, int filenum) const // //========================================================================== -int FileSystem::FileLength (int lump) const +ptrdiff_t FileSystem::FileLength (int lump) const { if ((size_t)lump >= NumEntries) { return -1; } - return FileInfo[lump].lump->LumpSize; -} - -//========================================================================== -// -// GetFileOffset -// -// Returns the offset from the beginning of the file to the lump. -// Returns -1 if the lump is compressed or can't be read directly -// -//========================================================================== - -int FileSystem::GetFileOffset (int lump) -{ - if ((size_t)lump >= NumEntries) - { - return -1; - } - return FileInfo[lump].lump->GetFileOffset(); + const auto &lump_p = FileInfo[lump]; + return (int)lump_p.resfile->Length(lump_p.resindex); } //========================================================================== @@ -810,7 +797,8 @@ int FileSystem::GetFileFlags (int lump) return 0; } - return FileInfo[lump].lump->Flags ^ FileInfo[lump].flags; + const auto& lump_p = FileInfo[lump]; + return lump_p.resfile->GetEntryFlags(lump_p.resindex) ^ lump_p.flags; } //========================================================================== @@ -906,8 +894,6 @@ void FileSystem::RenameFile(int num, const char* newfn) // //========================================================================== -static FResourceLump placeholderLump; - void FileSystem::MoveLumpsInFolder(const char *path) { if (FileInfo.size() == 0) @@ -925,11 +911,12 @@ void FileSystem::MoveLumpsInFolder(const char *path) if (li.rfnum >= GetIwadNum()) break; if (strnicmp(li.LongName, path, len) == 0) { - FileInfo.push_back(li); - li.lump = &placeholderLump; // Make the old entry point to something empty. We cannot delete the lump record here because it'd require adjustment of all indices in the list. + auto lic = li; // make a copy before pushing. + FileInfo.push_back(lic); + li.LongName = ""; //nuke the name of the old record. + li.shortName.qword = 0; auto &ln = FileInfo.back(); - ln.lump->LumpNameSetup(ln.LongName + len, stringpool); // may be able to avoid the string allocation! - ln.SetFromLump(rfnum, ln.lump, stringpool); + ln.SetFromLump(li.resfile, li.resindex, rfnum, stringpool, ln.LongName + len); } } } @@ -1309,9 +1296,7 @@ FileData FileSystem::ReadFile (int lump) { throw FileSystemException("ReadFile: %u >= NumEntries", lump); } - auto lumpp = FileInfo[lump].lump; - - return FileData(lumpp); + return FileInfo[lump].resfile->Read(FileInfo[lump].resindex); } //========================================================================== @@ -1323,53 +1308,31 @@ FileData FileSystem::ReadFile (int lump) //========================================================================== -FileReader FileSystem::OpenFileReader(int lump) +FileReader FileSystem::OpenFileReader(int lump, int readertype, int readerflags) { if ((unsigned)lump >= (unsigned)FileInfo.size()) { throw FileSystemException("OpenFileReader: %u >= NumEntries", lump); } - auto rl = FileInfo[lump].lump; - auto rd = rl->GetReader(); - - if (rl->RefCount == 0 && rd != nullptr && !rd->GetBuffer() && !(rl->Flags & LUMPF_COMPRESSED)) - { - FileReader rdr; - rdr.OpenFilePart(*rd, rl->GetFileOffset(), rl->LumpSize); - return rdr; - } - return rl->NewReader(); // This always gets a reader to the cache -} - -FileReader FileSystem::ReopenFileReader(int lump, bool alwayscache) -{ - if ((unsigned)lump >= (unsigned)FileInfo.size()) - { - throw FileSystemException("ReopenFileReader: %u >= NumEntries", lump); - } - - auto rl = FileInfo[lump].lump; - auto rd = rl->GetReader(); - - if (rl->RefCount == 0 && rd != nullptr && !rd->GetBuffer() && !alwayscache && !(rl->Flags & LUMPF_COMPRESSED)) - { - int fileno = GetFileContainer(lump); - const char *filename = GetResourceFileFullName(fileno); - FileReader fr; - if (fr.OpenFile(filename, rl->GetFileOffset(), rl->LumpSize)) - { - return fr; - } - } - return rl->NewReader(); // This always gets a reader to the cache + auto file = FileInfo[lump].resfile; + return file->GetEntryReader(FileInfo[lump].resindex, readertype, readerflags); } FileReader FileSystem::OpenFileReader(const char* name) { + FileReader fr; auto lump = CheckNumForFullName(name); - if (lump < 0) return FileReader(); - else return OpenFileReader(lump); + if (lump >= 0) fr = OpenFileReader(lump); + return fr; +} + +FileReader FileSystem::ReopenFileReader(const char* name, bool alwayscache) +{ + FileReader fr; + auto lump = CheckNumForFullName(name); + if (lump >= 0) fr = ReopenFileReader(lump, alwayscache); + return fr; } //========================================================================== @@ -1498,7 +1461,7 @@ bool FileSystem::CreatePathlessCopy(const char *name, int id, int /*flags*/) if (slash == nullptr) { - FileInfo[lump].flags = LUMPF_FULLPATH; + FileInfo[lump].flags = RESFF_FULLPATH; return true; // already is pathless. } @@ -1506,7 +1469,7 @@ bool FileSystem::CreatePathlessCopy(const char *name, int id, int /*flags*/) // just create a new reference to the original data with a different name. oldlump.LongName = slash + 1; oldlump.resourceId = id; - oldlump.flags = LUMPF_FULLPATH; + oldlump.flags = RESFF_FULLPATH; FileInfo.push_back(oldlump); return true; } diff --git a/source/common/filesystem/source/fs_stringpool.cpp b/source/common/filesystem/source/fs_stringpool.cpp index 1e4231139..316078cae 100644 --- a/source/common/filesystem/source/fs_stringpool.cpp +++ b/source/common/filesystem/source/fs_stringpool.cpp @@ -104,10 +104,11 @@ StringPool::Block *StringPool::AddBlock(size_t size) return mem; } -void *StringPool::iAlloc(size_t size) +void *StringPool::Alloc(size_t size) { Block *block; + size = (size + 7) & ~7; for (block = TopBlock; block != nullptr; block = block->NextBlock) { void *res = block->Alloc(size); @@ -122,7 +123,7 @@ void *StringPool::iAlloc(size_t size) const char* StringPool::Strdup(const char* str) { - char* p = (char*)iAlloc((strlen(str) + 8) & ~7 ); + char* p = (char*)Alloc(strlen(str) + 1); strcpy(p, str); return p; } diff --git a/source/common/filesystem/source/fs_stringpool.h b/source/common/filesystem/source/fs_stringpool.h index 3472cd792..199a3483b 100644 --- a/source/common/filesystem/source/fs_stringpool.h +++ b/source/common/filesystem/source/fs_stringpool.h @@ -12,12 +12,12 @@ private: public: ~StringPool(); const char* Strdup(const char*); + void* Alloc(size_t size); protected: struct Block; Block *AddBlock(size_t size); - void *iAlloc(size_t size); Block *TopBlock; Block *FreeBlocks; diff --git a/source/common/filesystem/source/resourcefile.cpp b/source/common/filesystem/source/resourcefile.cpp index 420755a1a..6b284cc28 100644 --- a/source/common/filesystem/source/resourcefile.cpp +++ b/source/common/filesystem/source/resourcefile.cpp @@ -35,13 +35,29 @@ */ #include -#include "resourcefile_internal.h" +#include "resourcefile.h" #include "md5.hpp" #include "fs_stringpool.h" #include "files_internal.h" +#include "unicode.h" +#include "fs_findfile.h" +#include "fs_decompress.h" +#include "wildcards.hpp" namespace FileSys { +// this is for restricting shared file readers to the main thread. +thread_local bool mainThread; +void SetMainThread() +{ + // only set the global flag on the first thread calling this. + static bool done = false; + if (!done) + { + mainThread = done = true; + } +} + std::string ExtractBaseName(const char* path, bool include_extension) { const char* src, * dot; @@ -78,195 +94,43 @@ void strReplace(std::string& str, const char *from, const char* to) } } -//========================================================================== -// -// File reader that reads from a lump's cache -// -//========================================================================== - -class FLumpReader : public MemoryReader -{ - FResourceLump *source; - -public: - FLumpReader(FResourceLump *src) - : MemoryReader(NULL, src->LumpSize), source(src) - { - src->Lock(); - bufptr = src->Cache; - } - - ~FLumpReader() - { - source->Unlock(); - } -}; - - -//========================================================================== -// -// Base class for resource lumps -// -//========================================================================== - -FResourceLump::~FResourceLump() -{ - if (Cache != NULL && RefCount >= 0) - { - delete [] Cache; - Cache = NULL; - } - Owner = NULL; -} - - -//========================================================================== -// -// Sets up the lump name information for anything not coming from a WAD file. -// -//========================================================================== - -void FResourceLump::LumpNameSetup(const char *iname, StringPool* allocator) -{ - // this causes interference with real Dehacked lumps. - if (!stricmp(iname, "dehacked.exe")) - { - iname = ""; - } - else if (allocator) - { - iname = allocator->Strdup(iname); - } - - FullName = iname; -} - //========================================================================== // // Checks for embedded resource files // //========================================================================== -static bool IsWadInFolder(const FResourceFile* const archive, const char* const resPath) +bool FResourceFile::IsFileInFolder(const char* const resPath) { // Checks a special case when was put in // directory inside - if (NULL == archive) - { - return false; - } - - const auto dirName = ExtractBaseName(archive->GetFileName()); + const auto dirName = ExtractBaseName(FileName); const auto fileName = ExtractBaseName(resPath, true); const std::string filePath = dirName + '/' + fileName; return 0 == stricmp(filePath.c_str(), resPath); } -void FResourceLump::CheckEmbedded(LumpFilterInfo* lfi) +void FResourceFile::CheckEmbedded(uint32_t entry, LumpFilterInfo* lfi) { // Checks for embedded archives - const char *c = strstr(FullName, ".wad"); - if (c && strlen(c) == 4 && (!strchr(FullName, '/') || IsWadInFolder(Owner, FullName))) + auto FullName = Entries[entry].FileName; + const char *c = strstr(FullName, ".wad"); // fixme: Use lfi for this. + if (c && strlen(c) == 4 && (!strchr(FullName, '/') || IsFileInFolder(FullName))) { - Flags |= LUMPF_EMBEDDED; + Entries[entry].Flags |= RESFF_EMBEDDED; } else if (lfi) for (auto& fstr : lfi->embeddings) { if (!stricmp(FullName, fstr.c_str())) { - Flags |= LUMPF_EMBEDDED; + Entries[entry].Flags |= RESFF_EMBEDDED; } } } -//========================================================================== -// -// this is just for completeness. For non-Zips only an uncompressed lump can -// be returned. -// -//========================================================================== - -FCompressedBuffer FResourceLump::GetRawData() -{ - FCompressedBuffer cbuf = { (unsigned)LumpSize, (unsigned)LumpSize, METHOD_STORED, 0, 0, new char[LumpSize] }; - memcpy(cbuf.mBuffer, Lock(), LumpSize); - Unlock(); - cbuf.mCRC32 = crc32(0, (uint8_t*)cbuf.mBuffer, LumpSize); - return cbuf; -} - - -//========================================================================== -// -// Returns the owner's FileReader if it can be used to access this lump -// -//========================================================================== - -FileReader *FResourceLump::GetReader() -{ - return NULL; -} - -//========================================================================== -// -// Returns a file reader to the lump's cache -// -//========================================================================== - -FileReader FResourceLump::NewReader() -{ - return FileReader(new FLumpReader(this)); -} - -//========================================================================== -// -// Caches a lump's content and increases the reference counter -// -//========================================================================== - -void *FResourceLump::Lock() -{ - if (Cache != NULL) - { - if (RefCount > 0) RefCount++; - } - else if (LumpSize > 0) - { - try - { - FillCache(); - } - catch (const FileSystemException& err) - { - // enrich the message with info about this lump. - throw FileSystemException("%s, file '%s': %s", getName(), Owner->GetFileName(), err.what()); - } - } - return Cache; -} - -//========================================================================== -// -// Decrements reference counter and frees lump if counter reaches 0 -// -//========================================================================== - -int FResourceLump::Unlock() -{ - if (LumpSize > 0 && RefCount > 0) - { - if (--RefCount == 0) - { - delete [] Cache; - Cache = NULL; - } - } - return RefCount; -} - //========================================================================== // // Opens a resource file @@ -282,11 +146,13 @@ FResourceFile *CheckPak(const char *filename, FileReader &file, LumpFilterInfo* FResourceFile *CheckZip(const char *filename, FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp); FResourceFile *Check7Z(const char *filename, FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp); FResourceFile* CheckSSI(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp); +FResourceFile* CheckHog(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp); +FResourceFile* CheckMvl(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp); FResourceFile* CheckWHRes(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp); FResourceFile *CheckLump(const char *filename,FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp); FResourceFile *CheckDir(const char *filename, bool nosub, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp); -static CheckFunc funcs[] = { CheckWad, CheckZip, Check7Z, CheckPak, CheckGRP, CheckRFF, CheckSSI, CheckWHRes, CheckLump }; +static CheckFunc funcs[] = { CheckWad, CheckZip, Check7Z, CheckPak, CheckGRP, CheckRFF, CheckSSI, CheckHog, CheckMvl, CheckWHRes, CheckLump }; static int nulPrintf(FSMessageLevel msg, const char* fmt, ...) { @@ -347,11 +213,95 @@ FResourceFile::~FResourceFile() if (!stringpool->shared) delete stringpool; } -int lumpcmp(const void * a, const void * b) +//========================================================================== +// +// this is just for completeness. For non-Zips only an uncompressed lump can +// be returned. +// +//========================================================================== + +FCompressedBuffer FResourceFile::GetRawData(uint32_t entry) { - FResourceLump * rec1 = (FResourceLump *)a; - FResourceLump * rec2 = (FResourceLump *)b; - return stricmp(rec1->getName(), rec2->getName()); + size_t LumpSize = entry << NumLumps ? Entries[entry].Length : 0; + FCompressedBuffer cbuf = { LumpSize, LumpSize, METHOD_STORED, 0, 0, LumpSize == 0? nullptr : new char[LumpSize] }; + if (LumpSize > 0) + { + auto fr = GetEntryReader(entry, READER_SHARED, 0); + size_t read = fr.Read(cbuf.mBuffer, LumpSize); + if (read < LumpSize) + { + delete cbuf.mBuffer; + cbuf.mBuffer = nullptr; + LumpSize = cbuf.mCompressedSize = cbuf.mSize = 0; + } + } + if (LumpSize > 0) + cbuf.mCRC32 = crc32(0, (uint8_t*)cbuf.mBuffer, LumpSize); + return cbuf; +} + + +//========================================================================== +// +// normalize the visible file name in the system +// to lowercase canonical precomposed Unicode. +// +//========================================================================== + +const char* FResourceFile::NormalizeFileName(const char* fn, int fallbackcp) +{ + if (!fn || !*fn) return ""; + auto norm = tolower_normalize(fn); + if (!norm) + { + if (fallbackcp == 437) + { + std::vector buffer; + ibm437_to_utf8(fn, buffer); + norm = tolower_normalize(buffer.data()); + } + // maybe handle other codepages + else + { + // if the filename is not valid UTF-8, nuke all bytes larger than 0x80 so that we still got something semi-usable + std::string ffn = fn; + for (auto& c : ffn) + { + if (c & 0x80) c = '@'; + } + norm = tolower_normalize(&ffn.front()); + } + } + FixPathSeparator(norm); + auto pooled = stringpool->Strdup(norm); + free(norm); + return pooled; + +} + +//========================================================================== +// +// allocate the Entries array +// this also uses the string pool to reduce maintenance +// +//========================================================================== + +FResourceEntry* FResourceFile::AllocateEntries(int count) +{ + NumLumps = count; + Entries = (FResourceEntry*)stringpool->Alloc(count * sizeof(FResourceEntry)); + memset(Entries, 0, count * sizeof(FResourceEntry)); + return Entries; +} + + +//--------------------------------------------------- +int entrycmp(const void* a, const void* b) +{ + FResourceEntry* rec1 = (FResourceEntry*)a; + FResourceEntry* rec2 = (FResourceEntry*)b; + // we are comparing lowercase UTF-8 here + return strcmp(rec1->FileName, rec2->FileName); } //========================================================================== @@ -400,30 +350,132 @@ void FResourceFile::GenerateHash() // //========================================================================== -void FResourceFile::PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterInfo *filter) +void FResourceFile::PostProcessArchive(LumpFilterInfo *filter) { - // Entries in archives are sorted alphabetically - qsort(lumps, NumLumps, lumpsize, lumpcmp); + // only do this for archive types which contain full file names. All others are assumed to be pre-sorted. + if (NumLumps == 0 || !(Entries[0].Flags & RESFF_FULLPATH)) return; + + // First eliminate all unwanted files + if (filter) + { + for (uint32_t i = 0; i < NumLumps; i++) + { + std::string name = Entries[i].FileName; + for (auto& pattern : filter->blockednames) + { + if (wildcards::match(name, pattern)) + { + Entries[i].FileName = ""; + continue; + } + } + } + } + + // Entries in archives are sorted alphabetically. + qsort(Entries, NumLumps, sizeof(Entries[0]), entrycmp); if (!filter) return; + FindCommonFolder(filter); // Filter out lumps using the same names as the Autoload.* sections - // in the ini file use. We reduce the maximum lump concidered after + // in the ini file. We reduce the maximum lump concidered after // each one so that we don't risk refiltering already filtered lumps. uint32_t max = NumLumps; - max -= FilterLumpsByGameType(filter, lumps, lumpsize, max); - ptrdiff_t len; - ptrdiff_t lastpos = -1; - std::string file; - std::string& LumpFilter = filter->dotFilter; - while (size_t(len = LumpFilter.find_first_of('.', lastpos+1)) != LumpFilter.npos) + for (auto& LumpFilter : filter->gameTypeFilter) { - max -= FilterLumps(std::string(LumpFilter, 0, len), lumps, lumpsize, max); - lastpos = len; + ptrdiff_t len; + ptrdiff_t lastpos = -1; + std::string file; + while (size_t(len = LumpFilter.find_first_of('.', lastpos + 1)) != LumpFilter.npos) + { + max -= FilterLumps(std::string(LumpFilter, 0, len), max); + lastpos = len; + } + max -= FilterLumps(LumpFilter, max); } - max -= FilterLumps(LumpFilter, lumps, lumpsize, max); - JunkLeftoverFilters(lumps, lumpsize, max); + JunkLeftoverFilters(max); + + for (uint32_t i = 0; i < NumLumps; i++) + { + CheckEmbedded(i, filter); + } + +} + +//========================================================================== +// +// FResourceFile :: FindCommonFolder +// +// Checks if all content is in a common folder that can be stripped out. +// +//========================================================================== + +void FResourceFile::FindCommonFolder(LumpFilterInfo* filter) +{ + std::string name0, name1; + bool foundspeciallump = false; + bool foundprefix = false; + + // try to find a path prefix. + for (uint32_t i = 0; i < NumLumps; i++) + { + if (*Entries[i].FileName == 0) continue; + std::string name = Entries[i].FileName; + + // first eliminate files we do not want to have. + // Some, like MacOS resource forks and executables are eliminated unconditionally, but the calling code can alsp pass a list of invalid content. + if (name.find("filter/") == 0) + return; // 'filter' is a reserved name of the file system. If this appears in the root we got no common folder, and 'filter' cannot be it. + + if (!foundprefix) + { + // check for special names, if one of these gets found this must be treated as a normal zip. + bool isspecial = name.find("/") == std::string::npos || + (filter && std::find(filter->reservedFolders.begin(), filter->reservedFolders.end(), name) != filter->reservedFolders.end()); + if (isspecial) break; + name0 = std::string(name, 0, name.rfind("/") + 1); + name1 = std::string(name, 0, name.find("/") + 1); + foundprefix = true; + } + + if (name.find(name0) != 0) + { + if (!name1.empty()) + { + name0 = name1; + if (name.find(name0) != 0) + { + name0 = ""; + } + } + if (name0.empty()) + break; + } + if (!foundspeciallump && filter) + { + // at least one of the more common definition lumps must be present. + for (auto& p : filter->requiredPrefixes) + { + if (name.find(name0 + p) == 0 || name.rfind(p) == size_t(name.length() - p.length())) + { + foundspeciallump = true; + break; + } + } + } + } + // If it ran through the list without finding anything it should not attempt any path remapping. + if (!foundspeciallump || name0.empty()) return; + + size_t pathlen = name0.length(); + for (uint32_t i = 0; i < NumLumps; i++) + { + if (Entries[i].FileName[0] == 0) continue; + Entries[i].FileName += pathlen; + + } } //========================================================================== @@ -436,7 +488,7 @@ void FResourceFile::PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterI // //========================================================================== -int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_t lumpsize, uint32_t max) +int FResourceFile::FilterLumps(const std::string& filtername, uint32_t max) { uint32_t start, end; @@ -446,44 +498,35 @@ int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_ } std::string filter = "filter/" + filtername + '/'; - bool found = FindPrefixRange(filter.c_str(), lumps, lumpsize, max, start, end); - - // Workaround for old Doom filter names (todo: move out of here.) - if (!found && filtername.find("doom.id.doom") == 0) - { - strReplace(filter, "doom.id.doom", "doom.doom"); - found = FindPrefixRange(filter.c_str(), lumps, lumpsize, max, start, end); - } + bool found = FindPrefixRange(filter.c_str(), max, start, end); if (found) { - void *from = (uint8_t *)lumps + start * lumpsize; // Remove filter prefix from every name - void *lump_p = from; - for (uint32_t i = start; i < end; ++i, lump_p = (uint8_t *)lump_p + lumpsize) + for (uint32_t i = start; i < end; ++i) { - FResourceLump *lump = (FResourceLump *)lump_p; - assert(strnicmp(lump->FullName, filter.c_str(), filter.length()) == 0); - lump->LumpNameSetup(lump->FullName + filter.length(), nullptr); + assert(strnicmp(Entries[i].FileName, filter.c_str(), filter.length()) == 0); + Entries[i].FileName += filter.length(); } // Move filtered lumps to the end of the lump list. - size_t count = (end - start) * lumpsize; - void *to = (uint8_t *)lumps + NumLumps * lumpsize - count; + size_t count = (end - start); + auto from = Entries + start; + auto to = Entries + NumLumps - count; assert (to >= from); if (from != to) { // Copy filtered lumps to a temporary buffer. - uint8_t *filteredlumps = new uint8_t[count]; - memcpy(filteredlumps, from, count); + auto filteredlumps = new FResourceEntry[count]; + memcpy(filteredlumps, from, count * sizeof(*Entries)); // Shift lumps left to make room for the filtered ones at the end. - memmove(from, (uint8_t *)from + count, (NumLumps - end) * lumpsize); + memmove(from, from + count, (NumLumps - end) * sizeof(*Entries)); // Copy temporary buffer to newly freed space. - memcpy(to, filteredlumps, count); + memcpy(to, filteredlumps, count * sizeof(*Entries)); delete[] filteredlumps; } @@ -491,29 +534,6 @@ int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_ return end - start; } -//========================================================================== -// -// FResourceFile :: FilterLumpsByGameType -// -// Matches any lumps that match "filter/game-/*". Includes -// inclusive gametypes like Raven. -// -//========================================================================== - -int FResourceFile::FilterLumpsByGameType(LumpFilterInfo *filter, void *lumps, size_t lumpsize, uint32_t max) -{ - if (filter == nullptr) - { - return 0; - } - int count = 0; - for (auto &fstring : filter->gameTypeFilter) - { - count += FilterLumps(fstring, lumps, lumpsize, max); - } - return count; -} - //========================================================================== // // FResourceFile :: JunkLeftoverFilters @@ -522,19 +542,17 @@ int FResourceFile::FilterLumpsByGameType(LumpFilterInfo *filter, void *lumps, si // //========================================================================== -void FResourceFile::JunkLeftoverFilters(void *lumps, size_t lumpsize, uint32_t max) +void FResourceFile::JunkLeftoverFilters(uint32_t max) { uint32_t start, end; - if (FindPrefixRange("filter/", lumps, lumpsize, max, start, end)) + if (FindPrefixRange("filter/", max, start, end)) { // Since the resource lumps may contain non-POD data besides the // full name, we "delete" them by erasing their names so they // can't be found. - void *stop = (uint8_t *)lumps + end * lumpsize; - for (void *p = (uint8_t *)lumps + start * lumpsize; p < stop; p = (uint8_t *)p + lumpsize) + for (uint32_t i = start; i < end; i++) { - FResourceLump *lump = (FResourceLump *)p; - lump->clearName(); + Entries[i].FileName = ""; } } } @@ -549,25 +567,24 @@ void FResourceFile::JunkLeftoverFilters(void *lumps, size_t lumpsize, uint32_t m // //========================================================================== -bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lumpsize, uint32_t maxlump, uint32_t &start, uint32_t &end) +bool FResourceFile::FindPrefixRange(const char* filter, uint32_t maxlump, uint32_t &start, uint32_t &end) { uint32_t min, max, mid, inside; - FResourceLump *lump; int cmp = 0; end = start = 0; // Pretend that our range starts at 1 instead of 0 so that we can avoid // unsigned overflow if the range starts at the first lump. - lumps = (uint8_t *)lumps - lumpsize; + auto lumps = &Entries[-1]; // Binary search to find any match at all. mid = min = 1, max = maxlump; while (min <= max) { mid = min + (max - min) / 2; - lump = (FResourceLump *)((uint8_t *)lumps + mid * lumpsize); - cmp = strnicmp(lump->FullName, filter, (int)strlen(filter)); + auto lump = &lumps[mid]; + cmp = strnicmp(lump->FileName, filter, strlen(filter)); if (cmp == 0) break; else if (cmp < 0) @@ -586,8 +603,8 @@ bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lump while (min <= max) { mid = min + (max - min) / 2; - lump = (FResourceLump *)((uint8_t *)lumps + mid * lumpsize); - cmp = strnicmp(lump->FullName, filter, (int)strlen(filter)); + auto lump = &lumps[mid]; + cmp = strnicmp(lump->FileName, filter, strlen(filter)); // Go left on matches and right on misses. if (cmp == 0) max = mid - 1; @@ -601,8 +618,8 @@ bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lump while (min <= max) { mid = min + (max - min) / 2; - lump = (FResourceLump *)((uint8_t *)lumps + mid * lumpsize); - cmp = strnicmp(lump->FullName, filter, (int)strlen(filter)); + auto lump = &lumps[mid]; + cmp = strnicmp(lump->FileName, filter, strlen(filter)); // Go right on matches and left on misses. if (cmp == 0) min = mid + 1; @@ -621,27 +638,19 @@ bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lump int FResourceFile::FindEntry(const char *name) { + auto norm_fn = tolower_normalize(name); for (unsigned i = 0; i < NumLumps; i++) { - if (!stricmp(name, getName(i))) + if (!strcmp(norm_fn, getName(i))) { + free(norm_fn); return i; } } + free(norm_fn); return -1; } -//========================================================================== -// -// Caches a lump's content and increases the reference counter -// -//========================================================================== - -FileReader *FUncompressedLump::GetReader() -{ - Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet); - return Owner->GetContainerReader(); -} //========================================================================== // @@ -649,103 +658,73 @@ FileReader *FUncompressedLump::GetReader() // //========================================================================== -int FUncompressedLump::FillCache() +FileReader FResourceFile::GetEntryReader(uint32_t entry, int readertype, int readerflags) { - const char * buffer = Owner->GetContainerReader()->GetBuffer(); - - if (buffer != NULL) + FileReader fr; + if (entry < NumLumps) { - // This is an in-memory file so the cache can point directly to the file's data. - Cache = const_cast(buffer) + Position; - RefCount = -1; - return -1; - } - - Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet); - Cache = new char[LumpSize]; - - auto read = Owner->GetContainerReader()->Read(Cache, LumpSize); - if (read != LumpSize) - { - throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize); - } - - RefCount = 1; - return 1; -} - -//========================================================================== -// -// Base class for uncompressed resource files -// -//========================================================================== - -FUncompressedFile::FUncompressedFile(const char *filename, StringPool* sp) -: FResourceFile(filename, sp) -{} - -FUncompressedFile::FUncompressedFile(const char *filename, FileReader &r, StringPool* sp) - : FResourceFile(filename, r, sp) -{} - - -//========================================================================== -// -// external lump -// -//========================================================================== - -FExternalLump::FExternalLump(const char *_filename, int filesize, StringPool* stringpool) -{ - FileName = stringpool->Strdup(_filename); - - if (filesize == -1) - { - FileReader f; - - if (f.OpenFile(_filename)) + if (Entries[entry].Flags & RESFF_NEEDFILESTART) { - LumpSize = (int)f.GetLength(); + SetEntryAddress(entry); + } + if (!(Entries[entry].Flags & RESFF_COMPRESSED)) + { + auto buf = Reader.GetBuffer(); + // if this is backed by a memory buffer, create a new reader directly referencing it. + if (buf != nullptr) + { + fr.OpenMemory(buf + Entries[entry].Position, Entries[entry].Length); + } + else + { + if (readertype == READER_SHARED && !mainThread) + readertype = READER_NEW; + if (readertype == READER_SHARED) + { + fr.OpenFilePart(Reader, Entries[entry].Position, Entries[entry].Length); + } + else if (readertype == READER_NEW) + { + fr.OpenFile(FileName, Entries[entry].Position, Entries[entry].Length); + } + else if (readertype == READER_CACHED) + { + Reader.Seek(Entries[entry].Position, FileReader::SeekSet); + auto data = Reader.Read(Entries[entry].Length); + fr.OpenMemoryArray(data); + } + } } else { - LumpSize = 0; + FileReader fri; + if (readertype == READER_NEW || !mainThread) fri.OpenFile(FileName, Entries[entry].Position, Entries[entry].CompressedSize); + else fri.OpenFilePart(Reader, Entries[entry].Position, Entries[entry].CompressedSize); + int flags = DCF_TRANSFEROWNER | DCF_EXCEPTIONS; + if (readertype == READER_CACHED) flags |= DCF_CACHED; + else if (readerflags & READERFLAG_SEEKABLE) flags |= DCF_SEEKABLE; + OpenDecompressor(fr, fri, Entries[entry].Length, Entries[entry].Method, flags); } } - else - { - LumpSize = filesize; - } + return fr; } - -//========================================================================== -// -// Caches a lump's content and increases the reference counter -// For external lumps this reopens the file each time it is accessed -// -//========================================================================== - -int FExternalLump::FillCache() +FileData FResourceFile::Read(int entry) { - Cache = new char[LumpSize]; - FileReader f; - - if (f.OpenFile(FileName)) + if (!(Entries[entry].Flags & RESFF_COMPRESSED)) { - auto read = f.Read(Cache, LumpSize); - if (read != LumpSize) + auto buf = Reader.GetBuffer(); + // if this is backed by a memory buffer, we can just return a reference to the backing store. + if (buf != nullptr) { - throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize); + return FileData(buf + Entries[entry].Position, Entries[entry].Length, false); } } - else - { - throw FileSystemException("unable to open file"); - } - RefCount = 1; - return 1; + + auto fr = GetEntryReader(entry, READER_SHARED, 0); + return fr.Read(entry < NumLumps ? Entries[entry].Length : 0); } + } diff --git a/source/common/filesystem/source/resourcefile_internal.h b/source/common/filesystem/source/resourcefile_internal.h deleted file mode 100644 index 4371a7460..000000000 --- a/source/common/filesystem/source/resourcefile_internal.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include "resourcefile.h" - -namespace FileSys { -struct FUncompressedLump : public FResourceLump -{ - int Position; - - virtual FileReader *GetReader(); - virtual int FillCache() override; - virtual int GetFileOffset() { return Position; } - -}; - -// Base class for uncompressed resource files (WAD, GRP, PAK and single lumps) -class FUncompressedFile : public FResourceFile -{ -protected: - TArray Lumps; - - FUncompressedFile(const char *filename, StringPool* sp); - FUncompressedFile(const char *filename, FileReader &r, StringPool* sp); - virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; } -}; - - -// should only be used internally. -struct FExternalLump : public FResourceLump -{ - const char* FileName; - - FExternalLump(const char *_filename, int filesize, StringPool* sp); - virtual int FillCache() override; - -}; - -} \ No newline at end of file diff --git a/source/common/filesystem/source/unicode.cpp b/source/common/filesystem/source/unicode.cpp index d102f1fc0..3d148270a 100644 --- a/source/common/filesystem/source/unicode.cpp +++ b/source/common/filesystem/source/unicode.cpp @@ -121,7 +121,7 @@ void ibm437_to_utf8(const char* in, std::vector& buffer) while (int char1 = (uint8_t)*in++) { - if (char1 >= 0x80) char1 = ibm437map[char1]; + if (char1 >= 0x80) char1 = ibm437map[char1 - 0x80]; utf8_encode(char1, buffer); } buffer.push_back(0); @@ -140,4 +140,22 @@ char *tolower_normalize(const char *str) return (char*)retval; } +//========================================================================== +// +// validates the string for proper UTF-8 +// +//========================================================================== + +bool unicode_validate(const char* str) +{ + while (*str != 0) + { + int cp; + auto result = utf8proc_iterate((const uint8_t*)str, -1, &cp); + if (result < 0) return false; + } + return true; +} + + } diff --git a/source/common/filesystem/source/unicode.h b/source/common/filesystem/source/unicode.h index 6df51668d..8783dbc71 100644 --- a/source/common/filesystem/source/unicode.h +++ b/source/common/filesystem/source/unicode.h @@ -6,7 +6,7 @@ namespace FileSys { void utf16_to_utf8(const unsigned short* in, std::vector& buffer); void ibm437_to_utf8(const char* in, std::vector& buffer); -int unicode_tolower(int c); char *tolower_normalize(const char *str); +bool unicode_validate(const char* str); } diff --git a/source/common/filesystem/source/wildcards.hpp b/source/common/filesystem/source/wildcards.hpp new file mode 100644 index 000000000..9b8e4eac6 --- /dev/null +++ b/source/common/filesystem/source/wildcards.hpp @@ -0,0 +1,1833 @@ +// THIS FILE HAS BEEN GENERATED AUTOMATICALLY. DO NOT EDIT DIRECTLY. +// Generated: 2019-03-08 09:59:35.958950200 +// Copyright Tomas Zeman 2018. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +#ifndef WILDCARDS_HPP +#define WILDCARDS_HPP +#define WILDCARDS_VERSION_MAJOR 1 +#define WILDCARDS_VERSION_MINOR 5 +#define WILDCARDS_VERSION_PATCH 0 +#ifndef WILDCARDS_CARDS_HPP +#define WILDCARDS_CARDS_HPP +#include +namespace wildcards +{ +template +struct cards +{ +constexpr cards(T a, T s, T e) +: anything{std::move(a)}, +single{std::move(s)}, +escape{std::move(e)}, +set_enabled{false}, +alt_enabled{false} +{ +} +constexpr cards(T a, T s, T e, T so, T sc, T sn, T ao, T ac, T ar) +: anything{std::move(a)}, +single{std::move(s)}, +escape{std::move(e)}, +set_enabled{true}, +set_open{std::move(so)}, +set_close{std::move(sc)}, +set_not{std::move(sn)}, +alt_enabled{true}, +alt_open{std::move(ao)}, +alt_close{std::move(ac)}, +alt_or{std::move(ar)} +{ +} +T anything; +T single; +T escape; +bool set_enabled; +T set_open; +T set_close; +T set_not; +bool alt_enabled; +T alt_open; +T alt_close; +T alt_or; +}; +enum class cards_type +{ +standard, +extended +}; +template <> +struct cards +{ +constexpr cards(cards_type type = cards_type::extended) +: set_enabled{type == cards_type::extended}, alt_enabled{type == cards_type::extended} +{ +} +constexpr cards(char a, char s, char e) +: anything{std::move(a)}, +single{std::move(s)}, +escape{std::move(e)}, +set_enabled{false}, +alt_enabled{false} +{ +} +constexpr cards(char a, char s, char e, char so, char sc, char sn, char ao, char ac, char ar) +: anything{std::move(a)}, +single{std::move(s)}, +escape{std::move(e)}, +set_enabled{true}, +set_open{std::move(so)}, +set_close{std::move(sc)}, +set_not{std::move(sn)}, +alt_enabled{true}, +alt_open{std::move(ao)}, +alt_close{std::move(ac)}, +alt_or{std::move(ar)} +{ +} +char anything{'*'}; +char single{'?'}; +char escape{'\\'}; +bool set_enabled{true}; +char set_open{'['}; +char set_close{']'}; +char set_not{'!'}; +bool alt_enabled{true}; +char alt_open{'('}; +char alt_close{')'}; +char alt_or{'|'}; +}; +template <> +struct cards +{ +constexpr cards(cards_type type = cards_type::extended) +: set_enabled{type == cards_type::extended}, alt_enabled{type == cards_type::extended} +{ +} +constexpr cards(char16_t a, char16_t s, char16_t e) +: anything{std::move(a)}, +single{std::move(s)}, +escape{std::move(e)}, +set_enabled{false}, +alt_enabled{false} +{ +} +constexpr cards(char16_t a, char16_t s, char16_t e, char16_t so, char16_t sc, char16_t sn, +char16_t ao, char16_t ac, char16_t ar) +: anything{std::move(a)}, +single{std::move(s)}, +escape{std::move(e)}, +set_enabled{true}, +set_open{std::move(so)}, +set_close{std::move(sc)}, +set_not{std::move(sn)}, +alt_enabled{true}, +alt_open{std::move(ao)}, +alt_close{std::move(ac)}, +alt_or{std::move(ar)} +{ +} +char16_t anything{u'*'}; +char16_t single{u'?'}; +char16_t escape{u'\\'}; +bool set_enabled{true}; +char16_t set_open{u'['}; +char16_t set_close{u']'}; +char16_t set_not{u'!'}; +bool alt_enabled{true}; +char16_t alt_open{u'('}; +char16_t alt_close{u')'}; +char16_t alt_or{u'|'}; +}; +template <> +struct cards +{ +constexpr cards(cards_type type = cards_type::extended) +: set_enabled{type == cards_type::extended}, alt_enabled{type == cards_type::extended} +{ +} +constexpr cards(char32_t a, char32_t s, char32_t e) +: anything{std::move(a)}, +single{std::move(s)}, +escape{std::move(e)}, +set_enabled{false}, +alt_enabled{false} +{ +} +constexpr cards(char32_t a, char32_t s, char32_t e, char32_t so, char32_t sc, char32_t sn, +char32_t ao, char32_t ac, char32_t ar) +: anything{std::move(a)}, +single{std::move(s)}, +escape{std::move(e)}, +set_enabled{true}, +set_open{std::move(so)}, +set_close{std::move(sc)}, +set_not{std::move(sn)}, +alt_enabled{true}, +alt_open{std::move(ao)}, +alt_close{std::move(ac)}, +alt_or{std::move(ar)} +{ +} +char32_t anything{U'*'}; +char32_t single{U'?'}; +char32_t escape{U'\\'}; +bool set_enabled{true}; +char32_t set_open{U'['}; +char32_t set_close{U']'}; +char32_t set_not{U'!'}; +bool alt_enabled{true}; +char32_t alt_open{U'('}; +char32_t alt_close{U')'}; +char32_t alt_or{U'|'}; +}; +template <> +struct cards +{ +constexpr cards(cards_type type = cards_type::extended) +: set_enabled{type == cards_type::extended}, alt_enabled{type == cards_type::extended} +{ +} +constexpr cards(wchar_t a, wchar_t s, wchar_t e) +: anything{std::move(a)}, +single{std::move(s)}, +escape{std::move(e)}, +set_enabled{false}, +alt_enabled{false} +{ +} +constexpr cards(wchar_t a, wchar_t s, wchar_t e, wchar_t so, wchar_t sc, wchar_t sn, wchar_t ao, +wchar_t ac, wchar_t ar) +: anything{std::move(a)}, +single{std::move(s)}, +escape{std::move(e)}, +set_enabled{true}, +set_open{std::move(so)}, +set_close{std::move(sc)}, +set_not{std::move(sn)}, +alt_enabled{true}, +alt_open{std::move(ao)}, +alt_close{std::move(ac)}, +alt_or{std::move(ar)} +{ +} +wchar_t anything{L'*'}; +wchar_t single{L'?'}; +wchar_t escape{L'\\'}; +bool set_enabled{true}; +wchar_t set_open{L'['}; +wchar_t set_close{L']'}; +wchar_t set_not{L'!'}; +bool alt_enabled{true}; +wchar_t alt_open{L'('}; +wchar_t alt_close{L')'}; +wchar_t alt_or{L'|'}; +}; +template +constexpr cards make_cards(T&& a, T&& s, T&& e) +{ +return {std::forward(a), std::forward(s), std::forward(e)}; +} +template +constexpr cards make_cards(T&& a, T&& s, T&& e, T&& so, T&& sc, T&& sn, T&& ao, T&& ac, T&& ar) +{ +return {std::forward(a), std::forward(s), std::forward(e), +std::forward(so), std::forward(sc), std::forward(sn), +std::forward(ao), std::forward(ac), std::forward(ar)}; +} +} +#endif +#ifndef WILDCARDS_MATCH_HPP +#define WILDCARDS_MATCH_HPP +#include +#include +#include +#ifndef CONFIG_HPP +#define CONFIG_HPP +#ifndef QUICKCPPLIB_HAS_FEATURE_H +#define QUICKCPPLIB_HAS_FEATURE_H +#if __cplusplus >= 201103L +#if !defined(__cpp_alias_templates) +#define __cpp_alias_templates 190000 +#endif +#if !defined(__cpp_attributes) +#define __cpp_attributes 190000 +#endif +#if !defined(__cpp_constexpr) +#if __cplusplus >= 201402L +#define __cpp_constexpr 201304 +#else +#define __cpp_constexpr 190000 +#endif +#endif +#if !defined(__cpp_decltype) +#define __cpp_decltype 190000 +#endif +#if !defined(__cpp_delegating_constructors) +#define __cpp_delegating_constructors 190000 +#endif +#if !defined(__cpp_explicit_conversion) +#define __cpp_explicit_conversion 190000 +#endif +#if !defined(__cpp_inheriting_constructors) +#define __cpp_inheriting_constructors 190000 +#endif +#if !defined(__cpp_initializer_lists) +#define __cpp_initializer_lists 190000 +#endif +#if !defined(__cpp_lambdas) +#define __cpp_lambdas 190000 +#endif +#if !defined(__cpp_nsdmi) +#define __cpp_nsdmi 190000 +#endif +#if !defined(__cpp_range_based_for) +#define __cpp_range_based_for 190000 +#endif +#if !defined(__cpp_raw_strings) +#define __cpp_raw_strings 190000 +#endif +#if !defined(__cpp_ref_qualifiers) +#define __cpp_ref_qualifiers 190000 +#endif +#if !defined(__cpp_rvalue_references) +#define __cpp_rvalue_references 190000 +#endif +#if !defined(__cpp_static_assert) +#define __cpp_static_assert 190000 +#endif +#if !defined(__cpp_unicode_characters) +#define __cpp_unicode_characters 190000 +#endif +#if !defined(__cpp_unicode_literals) +#define __cpp_unicode_literals 190000 +#endif +#if !defined(__cpp_user_defined_literals) +#define __cpp_user_defined_literals 190000 +#endif +#if !defined(__cpp_variadic_templates) +#define __cpp_variadic_templates 190000 +#endif +#endif +#if __cplusplus >= 201402L +#if !defined(__cpp_aggregate_nsdmi) +#define __cpp_aggregate_nsdmi 190000 +#endif +#if !defined(__cpp_binary_literals) +#define __cpp_binary_literals 190000 +#endif +#if !defined(__cpp_decltype_auto) +#define __cpp_decltype_auto 190000 +#endif +#if !defined(__cpp_generic_lambdas) +#define __cpp_generic_lambdas 190000 +#endif +#if !defined(__cpp_init_captures) +#define __cpp_init_captures 190000 +#endif +#if !defined(__cpp_return_type_deduction) +#define __cpp_return_type_deduction 190000 +#endif +#if !defined(__cpp_sized_deallocation) +#define __cpp_sized_deallocation 190000 +#endif +#if !defined(__cpp_variable_templates) +#define __cpp_variable_templates 190000 +#endif +#endif +#if defined(_MSC_VER) && !defined(__clang__) +#if !defined(__cpp_exceptions) && defined(_CPPUNWIND) +#define __cpp_exceptions 190000 +#endif +#if !defined(__cpp_rtti) && defined(_CPPRTTI) +#define __cpp_rtti 190000 +#endif +#if !defined(__cpp_alias_templates) && _MSC_VER >= 1800 +#define __cpp_alias_templates 190000 +#endif +#if !defined(__cpp_attributes) +#define __cpp_attributes 190000 +#endif +#if !defined(__cpp_constexpr) && _MSC_FULL_VER >= 190023506 +#define __cpp_constexpr 190000 +#endif +#if !defined(__cpp_decltype) && _MSC_VER >= 1600 +#define __cpp_decltype 190000 +#endif +#if !defined(__cpp_delegating_constructors) && _MSC_VER >= 1800 +#define __cpp_delegating_constructors 190000 +#endif +#if !defined(__cpp_explicit_conversion) && _MSC_VER >= 1800 +#define __cpp_explicit_conversion 190000 +#endif +#if !defined(__cpp_inheriting_constructors) && _MSC_VER >= 1900 +#define __cpp_inheriting_constructors 190000 +#endif +#if !defined(__cpp_initializer_lists) && _MSC_VER >= 1900 +#define __cpp_initializer_lists 190000 +#endif +#if !defined(__cpp_lambdas) && _MSC_VER >= 1600 +#define __cpp_lambdas 190000 +#endif +#if !defined(__cpp_nsdmi) && _MSC_VER >= 1900 +#define __cpp_nsdmi 190000 +#endif +#if !defined(__cpp_range_based_for) && _MSC_VER >= 1700 +#define __cpp_range_based_for 190000 +#endif +#if !defined(__cpp_raw_strings) && _MSC_VER >= 1800 +#define __cpp_raw_strings 190000 +#endif +#if !defined(__cpp_ref_qualifiers) && _MSC_VER >= 1900 +#define __cpp_ref_qualifiers 190000 +#endif +#if !defined(__cpp_rvalue_references) && _MSC_VER >= 1600 +#define __cpp_rvalue_references 190000 +#endif +#if !defined(__cpp_static_assert) && _MSC_VER >= 1600 +#define __cpp_static_assert 190000 +#endif +#if !defined(__cpp_user_defined_literals) && _MSC_VER >= 1900 +#define __cpp_user_defined_literals 190000 +#endif +#if !defined(__cpp_variadic_templates) && _MSC_VER >= 1800 +#define __cpp_variadic_templates 190000 +#endif +#if !defined(__cpp_binary_literals) && _MSC_VER >= 1900 +#define __cpp_binary_literals 190000 +#endif +#if !defined(__cpp_decltype_auto) && _MSC_VER >= 1900 +#define __cpp_decltype_auto 190000 +#endif +#if !defined(__cpp_generic_lambdas) && _MSC_VER >= 1900 +#define __cpp_generic_lambdas 190000 +#endif +#if !defined(__cpp_init_captures) && _MSC_VER >= 1900 +#define __cpp_init_captures 190000 +#endif +#if !defined(__cpp_return_type_deduction) && _MSC_VER >= 1900 +#define __cpp_return_type_deduction 190000 +#endif +#if !defined(__cpp_sized_deallocation) && _MSC_VER >= 1900 +#define __cpp_sized_deallocation 190000 +#endif +#if !defined(__cpp_variable_templates) && _MSC_FULL_VER >= 190023506 +#define __cpp_variable_templates 190000 +#endif +#endif +#if(defined(__GNUC__) && !defined(__clang__)) +#define QUICKCPPLIB_GCC (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) +#if !defined(__cpp_exceptions) && defined(__EXCEPTIONS) +#define __cpp_exceptions 190000 +#endif +#if !defined(__cpp_rtti) && defined(__GXX_RTTI) +#define __cpp_rtti 190000 +#endif +#if defined(__GXX_EXPERIMENTAL_CXX0X__) +#if !defined(__cpp_alias_templates) && (QUICKCPPLIB_GCC >= 40700) +#define __cpp_alias_templates 190000 +#endif +#if !defined(__cpp_attributes) && (QUICKCPPLIB_GCC >= 40800) +#define __cpp_attributes 190000 +#endif +#if !defined(__cpp_constexpr) && (QUICKCPPLIB_GCC >= 40600) +#define __cpp_constexpr 190000 +#endif +#if !defined(__cpp_decltype) && (QUICKCPPLIB_GCC >= 40300) +#define __cpp_decltype 190000 +#endif +#if !defined(__cpp_delegating_constructors) && (QUICKCPPLIB_GCC >= 40700) +#define __cpp_delegating_constructors 190000 +#endif +#if !defined(__cpp_explicit_conversion) && (QUICKCPPLIB_GCC >= 40500) +#define __cpp_explicit_conversion 190000 +#endif +#if !defined(__cpp_inheriting_constructors) && (QUICKCPPLIB_GCC >= 40800) +#define __cpp_inheriting_constructors 190000 +#endif +#if !defined(__cpp_initializer_lists) && (QUICKCPPLIB_GCC >= 40800) +#define __cpp_initializer_lists 190000 +#endif +#if !defined(__cpp_lambdas) && (QUICKCPPLIB_GCC >= 40500) +#define __cpp_lambdas 190000 +#endif +#if !defined(__cpp_nsdmi) && (QUICKCPPLIB_GCC >= 40700) +#define __cpp_nsdmi 190000 +#endif +#if !defined(__cpp_range_based_for) && (QUICKCPPLIB_GCC >= 40600) +#define __cpp_range_based_for 190000 +#endif +#if !defined(__cpp_raw_strings) && (QUICKCPPLIB_GCC >= 40500) +#define __cpp_raw_strings 190000 +#endif +#if !defined(__cpp_ref_qualifiers) && (QUICKCPPLIB_GCC >= 40801) +#define __cpp_ref_qualifiers 190000 +#endif +#if !defined(__cpp_rvalue_references) && defined(__cpp_rvalue_reference) +#define __cpp_rvalue_references __cpp_rvalue_reference +#endif +#if !defined(__cpp_static_assert) && (QUICKCPPLIB_GCC >= 40300) +#define __cpp_static_assert 190000 +#endif +#if !defined(__cpp_unicode_characters) && (QUICKCPPLIB_GCC >= 40500) +#define __cpp_unicode_characters 190000 +#endif +#if !defined(__cpp_unicode_literals) && (QUICKCPPLIB_GCC >= 40500) +#define __cpp_unicode_literals 190000 +#endif +#if !defined(__cpp_user_defined_literals) && (QUICKCPPLIB_GCC >= 40700) +#define __cpp_user_defined_literals 190000 +#endif +#if !defined(__cpp_variadic_templates) && (QUICKCPPLIB_GCC >= 40400) +#define __cpp_variadic_templates 190000 +#endif +#endif +#endif +#if defined(__clang__) +#define QUICKCPPLIB_CLANG (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) +#if !defined(__cpp_exceptions) && (defined(__EXCEPTIONS) || defined(_CPPUNWIND)) +#define __cpp_exceptions 190000 +#endif +#if !defined(__cpp_rtti) && (defined(__GXX_RTTI) || defined(_CPPRTTI)) +#define __cpp_rtti 190000 +#endif +#if defined(__GXX_EXPERIMENTAL_CXX0X__) +#if !defined(__cpp_alias_templates) && (QUICKCPPLIB_CLANG >= 30000) +#define __cpp_alias_templates 190000 +#endif +#if !defined(__cpp_attributes) && (QUICKCPPLIB_CLANG >= 30300) +#define __cpp_attributes 190000 +#endif +#if !defined(__cpp_constexpr) && (QUICKCPPLIB_CLANG >= 30100) +#define __cpp_constexpr 190000 +#endif +#if !defined(__cpp_decltype) && (QUICKCPPLIB_CLANG >= 20900) +#define __cpp_decltype 190000 +#endif +#if !defined(__cpp_delegating_constructors) && (QUICKCPPLIB_CLANG >= 30000) +#define __cpp_delegating_constructors 190000 +#endif +#if !defined(__cpp_explicit_conversion) && (QUICKCPPLIB_CLANG >= 30000) +#define __cpp_explicit_conversion 190000 +#endif +#if !defined(__cpp_inheriting_constructors) && (QUICKCPPLIB_CLANG >= 30300) +#define __cpp_inheriting_constructors 190000 +#endif +#if !defined(__cpp_initializer_lists) && (QUICKCPPLIB_CLANG >= 30100) +#define __cpp_initializer_lists 190000 +#endif +#if !defined(__cpp_lambdas) && (QUICKCPPLIB_CLANG >= 30100) +#define __cpp_lambdas 190000 +#endif +#if !defined(__cpp_nsdmi) && (QUICKCPPLIB_CLANG >= 30000) +#define __cpp_nsdmi 190000 +#endif +#if !defined(__cpp_range_based_for) && (QUICKCPPLIB_CLANG >= 30000) +#define __cpp_range_based_for 190000 +#endif +#if !defined(__cpp_raw_strings) && defined(__cpp_raw_string_literals) +#define __cpp_raw_strings __cpp_raw_string_literals +#endif +#if !defined(__cpp_raw_strings) && (QUICKCPPLIB_CLANG >= 30000) +#define __cpp_raw_strings 190000 +#endif +#if !defined(__cpp_ref_qualifiers) && (QUICKCPPLIB_CLANG >= 20900) +#define __cpp_ref_qualifiers 190000 +#endif +#if !defined(__cpp_rvalue_references) && defined(__cpp_rvalue_reference) +#define __cpp_rvalue_references __cpp_rvalue_reference +#endif +#if !defined(__cpp_rvalue_references) && (QUICKCPPLIB_CLANG >= 20900) +#define __cpp_rvalue_references 190000 +#endif +#if !defined(__cpp_static_assert) && (QUICKCPPLIB_CLANG >= 20900) +#define __cpp_static_assert 190000 +#endif +#if !defined(__cpp_unicode_characters) && (QUICKCPPLIB_CLANG >= 30000) +#define __cpp_unicode_characters 190000 +#endif +#if !defined(__cpp_unicode_literals) && (QUICKCPPLIB_CLANG >= 30000) +#define __cpp_unicode_literals 190000 +#endif +#if !defined(__cpp_user_defined_literals) && defined(__cpp_user_literals) +#define __cpp_user_defined_literals __cpp_user_literals +#endif +#if !defined(__cpp_user_defined_literals) && (QUICKCPPLIB_CLANG >= 30100) +#define __cpp_user_defined_literals 190000 +#endif +#if !defined(__cpp_variadic_templates) && (QUICKCPPLIB_CLANG >= 20900) +#define __cpp_variadic_templates 190000 +#endif +#endif +#endif +#endif +#define cfg_HAS_CONSTEXPR14 (__cpp_constexpr >= 201304) +#if cfg_HAS_CONSTEXPR14 +#define cfg_constexpr14 constexpr +#else +#define cfg_constexpr14 +#endif +#if cfg_HAS_CONSTEXPR14 && defined(__clang__) +#define cfg_HAS_FULL_FEATURED_CONSTEXPR14 1 +#else +#define cfg_HAS_FULL_FEATURED_CONSTEXPR14 1 +#endif +#endif +#ifndef CX_FUNCTIONAL_HPP +#define CX_FUNCTIONAL_HPP +#include +namespace cx +{ +template +struct less +{ +constexpr auto operator()(const T& lhs, const T& rhs) const -> decltype(lhs < rhs) +{ +return lhs < rhs; +} +}; +template <> +struct less +{ +template +constexpr auto operator()(T&& lhs, U&& rhs) const +-> decltype(std::forward(lhs) < std::forward(rhs)) +{ +return std::forward(lhs) < std::forward(rhs); +} +}; +template +struct equal_to +{ +constexpr auto operator()(const T& lhs, const T& rhs) const -> decltype(lhs == rhs) +{ +return lhs == rhs; +} +}; +template <> +struct equal_to +{ +template +constexpr auto operator()(T&& lhs, U&& rhs) const +-> decltype(std::forward(lhs) == std::forward(rhs)) +{ +return std::forward(lhs) == std::forward(rhs); +} +}; +} +#endif +#ifndef CX_ITERATOR_HPP +#define CX_ITERATOR_HPP +#include +#include +namespace cx +{ +template +constexpr It next(It it) +{ +return it + 1; +} +template +constexpr It prev(It it) +{ +return it - 1; +} +template +constexpr auto size(const C& c) -> decltype(c.size()) +{ +return c.size(); +} +template +constexpr std::size_t size(const T (&)[N]) +{ +return N; +} +template +constexpr auto empty(const C& c) -> decltype(c.empty()) +{ +return c.empty(); +} +template +constexpr bool empty(const T (&)[N]) +{ +return false; +} +template +constexpr bool empty(std::initializer_list il) +{ +return il.size() == 0; +} +template +constexpr auto begin(const C& c) -> decltype(c.begin()) +{ +return c.begin(); +} +template +constexpr auto begin(C& c) -> decltype(c.begin()) +{ +return c.begin(); +} +template +constexpr T* begin(T (&array)[N]) +{ +return &array[0]; +} +template +constexpr const E* begin(std::initializer_list il) +{ +return il.begin(); +} +template +constexpr auto cbegin(const C& c) -> decltype(cx::begin(c)) +{ +return cx::begin(c); +} +template +constexpr auto end(const C& c) -> decltype(c.end()) +{ +return c.end(); +} +template +constexpr auto end(C& c) -> decltype(c.end()) +{ +return c.end(); +} +template +constexpr T* end(T (&array)[N]) +{ +return &array[N]; +} +template +constexpr const E* end(std::initializer_list il) +{ +return il.end(); +} +template +constexpr auto cend(const C& c) -> decltype(cx::end(c)) +{ +return cx::end(c); +} +} +#endif +#ifndef WILDCARDS_UTILITY_HPP +#define WILDCARDS_UTILITY_HPP +#include +#include +namespace wildcards +{ +template +struct const_iterator +{ +using type = typename std::remove_cv< +typename std::remove_reference()))>::type>::type; +}; +template +using const_iterator_t = typename const_iterator::type; +template +struct iterator +{ +using type = typename std::remove_cv< +typename std::remove_reference()))>::type>::type; +}; +template +using iterator_t = typename iterator::type; +template +struct iterated_item +{ +using type = typename std::remove_cv< +typename std::remove_reference())>::type>::type; +}; +template +using iterated_item_t = typename iterated_item::type; +template +struct container_item +{ +using type = typename std::remove_cv< +typename std::remove_reference()))>::type>::type; +}; +template +using container_item_t = typename container_item::type; +} +#endif +namespace wildcards +{ +template +struct full_match_result +{ +bool res; +SequenceIterator s, send, s1; +PatternIterator p, pend, p1; +constexpr operator bool() const +{ +return res; +} +}; +namespace detail +{ +template +struct match_result +{ +bool res; +SequenceIterator s; +PatternIterator p; +constexpr operator bool() const +{ +return res; +} +}; +template +constexpr match_result make_match_result(bool res, +SequenceIterator s, +PatternIterator p) +{ +return {std::move(res), std::move(s), std::move(p)}; +} +template +constexpr full_match_result make_full_match_result( +SequenceIterator s, SequenceIterator send, PatternIterator p, PatternIterator pend, +match_result mr) +{ +return {std::move(mr.res), std::move(s), std::move(send), std::move(mr.s), +std::move(p), std::move(pend), std::move(mr.p)}; +} +#if !cfg_HAS_FULL_FEATURED_CONSTEXPR14 +constexpr bool throw_invalid_argument(const char* what_arg) +{ +return what_arg == nullptr ? false : throw std::invalid_argument(what_arg); +} +template +constexpr T throw_invalid_argument(T t, const char* what_arg) +{ +return what_arg == nullptr ? t : throw std::invalid_argument(what_arg); +} +constexpr bool throw_logic_error(const char* what_arg) +{ +return what_arg == nullptr ? false : throw std::logic_error(what_arg); +} +template +constexpr T throw_logic_error(T t, const char* what_arg) +{ +return what_arg == nullptr ? t : throw std::logic_error(what_arg); +} +#endif +enum class is_set_state +{ +open, +not_or_first, +first, +next +}; +template +constexpr bool is_set( +PatternIterator p, PatternIterator pend, +const cards>& c = cards>(), +is_set_state state = is_set_state::open) +{ +#if cfg_HAS_CONSTEXPR14 +if (!c.set_enabled) +{ +return false; +} +while (p != pend) +{ +switch (state) +{ +case is_set_state::open: +if (*p != c.set_open) +{ +return false; +} +state = is_set_state::not_or_first; +break; +case is_set_state::not_or_first: +if (*p == c.set_not) +{ +state = is_set_state::first; +} +else +{ +state = is_set_state::next; +} +break; +case is_set_state::first: +state = is_set_state::next; +break; +case is_set_state::next: +if (*p == c.set_close) +{ +return true; +} +break; +default: +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::logic_error( +"The program execution should never end up here throwing this exception"); +#else +return throw_logic_error( +"The program execution should never end up here throwing this exception"); +#endif +} +p = cx::next(p); +} +return false; +#else +return c.set_enabled && p != pend && +(state == is_set_state::open +? *p == c.set_open && is_set(cx::next(p), pend, c, is_set_state::not_or_first) +: +state == is_set_state::not_or_first +? *p == c.set_not ? is_set(cx::next(p), pend, c, is_set_state::first) +: is_set(cx::next(p), pend, c, is_set_state::next) +: state == is_set_state::first +? is_set(cx::next(p), pend, c, is_set_state::next) +: state == is_set_state::next +? *p == c.set_close || +is_set(cx::next(p), pend, c, is_set_state::next) +: throw std::logic_error("The program execution should never end up " +"here throwing this exception")); +#endif +} +enum class set_end_state +{ +open, +not_or_first, +first, +next +}; +template +constexpr PatternIterator set_end( +PatternIterator p, PatternIterator pend, +const cards>& c = cards>(), +set_end_state state = set_end_state::open) +{ +#if cfg_HAS_CONSTEXPR14 +if (!c.set_enabled) +{ +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::invalid_argument("The use of sets is disabled"); +#else +return throw_invalid_argument(p, "The use of sets is disabled"); +#endif +} +while (p != pend) +{ +switch (state) +{ +case set_end_state::open: +if (*p != c.set_open) +{ +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::invalid_argument("The given pattern is not a valid set"); +#else +return throw_invalid_argument(p, "The given pattern is not a valid set"); +#endif +} +state = set_end_state::not_or_first; +break; +case set_end_state::not_or_first: +if (*p == c.set_not) +{ +state = set_end_state::first; +} +else +{ +state = set_end_state::next; +} +break; +case set_end_state::first: +state = set_end_state::next; +break; +case set_end_state::next: +if (*p == c.set_close) +{ +return cx::next(p); +} +break; +default: +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::logic_error( +"The program execution should never end up here throwing this exception"); +#else +return throw_logic_error( +p, "The program execution should never end up here throwing this exception"); +#endif +} +p = cx::next(p); +} +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::invalid_argument("The given pattern is not a valid set"); +#else +return throw_invalid_argument(p, "The given pattern is not a valid set"); +#endif +#else +return !c.set_enabled +? throw std::invalid_argument("The use of sets is disabled") +: p == pend +? throw std::invalid_argument("The given pattern is not a valid set") +: +state == set_end_state::open +? *p == c.set_open +? set_end(cx::next(p), pend, c, set_end_state::not_or_first) +: throw std::invalid_argument("The given pattern is not a valid set") +: +state == set_end_state::not_or_first +? *p == c.set_not ? set_end(cx::next(p), pend, c, set_end_state::first) +: set_end(cx::next(p), pend, c, set_end_state::next) +: state == set_end_state::first +? set_end(cx::next(p), pend, c, set_end_state::next) +: state == set_end_state::next +? *p == c.set_close +? cx::next(p) +: set_end(cx::next(p), pend, c, set_end_state::next) +: throw std::logic_error( +"The program execution should never end up " +"here throwing this exception"); +#endif +} +enum class match_set_state +{ +open, +not_or_first_in, +first_out, +next_in, +next_out +}; +template > +constexpr match_result match_set( +SequenceIterator s, SequenceIterator send, PatternIterator p, PatternIterator pend, +const cards>& c = cards>(), +const EqualTo& equal_to = EqualTo(), match_set_state state = match_set_state::open) +{ +#if cfg_HAS_CONSTEXPR14 +if (!c.set_enabled) +{ +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::invalid_argument("The use of sets is disabled"); +#else +return throw_invalid_argument(make_match_result(false, s, p), "The use of sets is disabled"); +#endif +} +while (p != pend) +{ +switch (state) +{ +case match_set_state::open: +if (*p != c.set_open) +{ +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::invalid_argument("The given pattern is not a valid set"); +#else +return throw_invalid_argument(make_match_result(false, s, p), +"The given pattern is not a valid set"); +#endif +} +state = match_set_state::not_or_first_in; +break; +case match_set_state::not_or_first_in: +if (*p == c.set_not) +{ +state = match_set_state::first_out; +} +else +{ +if (s == send) +{ +return make_match_result(false, s, p); +} +if (equal_to(*s, *p)) +{ +return make_match_result(true, s, p); +} +state = match_set_state::next_in; +} +break; +case match_set_state::first_out: +if (s == send || equal_to(*s, *p)) +{ +return make_match_result(false, s, p); +} +state = match_set_state::next_out; +break; +case match_set_state::next_in: +if (*p == c.set_close || s == send) +{ +return make_match_result(false, s, p); +} +if (equal_to(*s, *p)) +{ +return make_match_result(true, s, p); +} +break; +case match_set_state::next_out: +if (*p == c.set_close) +{ +return make_match_result(true, s, p); +} +if (s == send || equal_to(*s, *p)) +{ +return make_match_result(false, s, p); +} +break; +default: +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::logic_error( +"The program execution should never end up here throwing this exception"); +#else +return throw_logic_error( +make_match_result(false, s, p), +"The program execution should never end up here throwing this exception"); +#endif +} +p = cx::next(p); +} +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::invalid_argument("The given pattern is not a valid set"); +#else +return throw_invalid_argument(make_match_result(false, s, p), +"The given pattern is not a valid set"); +#endif +#else +return !c.set_enabled +? throw std::invalid_argument("The use of sets is disabled") +: p == pend +? throw std::invalid_argument("The given pattern is not a valid set") +: state == match_set_state::open +? *p == c.set_open +? match_set(s, send, cx::next(p), pend, c, equal_to, +match_set_state::not_or_first_in) +: +throw std::invalid_argument("The given pattern is not a valid set") +: +state == match_set_state::not_or_first_in +? *p == c.set_not +? match_set(s, send, cx::next(p), pend, c, equal_to, +match_set_state::first_out) +: +s == send ? make_match_result(false, s, p) +: equal_to(*s, *p) +? make_match_result(true, s, p) +: match_set(s, send, cx::next(p), pend, c, +equal_to, match_set_state::next_in) +: +state == match_set_state::first_out +? s == send || equal_to(*s, *p) +? make_match_result(false, s, p) +: match_set(s, send, cx::next(p), pend, c, equal_to, +match_set_state::next_out) +: +state == match_set_state::next_in +? *p == c.set_close || s == send +? make_match_result(false, s, p) +: equal_to(*s, *p) ? make_match_result(true, s, p) +: match_set(s, send, cx::next(p), +pend, c, equal_to, state) +: +state == match_set_state::next_out +? *p == c.set_close +? make_match_result(true, s, p) +: s == send || equal_to(*s, *p) +? make_match_result(false, s, p) +: match_set(s, send, cx::next(p), pend, c, +equal_to, state) +: throw std::logic_error( +"The program execution should never end up " +"here " +"throwing this exception"); +#endif +} +enum class is_alt_state +{ +open, +next, +escape +}; +template +constexpr bool is_alt( +PatternIterator p, PatternIterator pend, +const cards>& c = cards>(), +is_alt_state state = is_alt_state::open, int depth = 0) +{ +#if cfg_HAS_CONSTEXPR14 +if (!c.alt_enabled) +{ +return false; +} +while (p != pend) +{ +switch (state) +{ +case is_alt_state::open: +if (*p != c.alt_open) +{ +return false; +} +state = is_alt_state::next; +++depth; +break; +case is_alt_state::next: +if (*p == c.escape) +{ +state = is_alt_state::escape; +} +else if (c.set_enabled && *p == c.set_open && +is_set(cx::next(p), pend, c, is_set_state::not_or_first)) +{ +p = cx::prev(set_end(cx::next(p), pend, c, set_end_state::not_or_first)); +} +else if (*p == c.alt_open) +{ +++depth; +} +else if (*p == c.alt_close) +{ +--depth; +if (depth == 0) +{ +return true; +} +} +break; +case is_alt_state::escape: +state = is_alt_state::next; +break; +default: +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::logic_error( +"The program execution should never end up here throwing this exception"); +#else +return throw_logic_error( +p, "The program execution should never end up here throwing this exception"); +#endif +} +p = cx::next(p); +} +return false; +#else +return c.alt_enabled && p != pend && +(state == is_alt_state::open +? *p == c.alt_open && is_alt(cx::next(p), pend, c, is_alt_state::next, depth + 1) +: state == is_alt_state::next +? *p == c.escape +? is_alt(cx::next(p), pend, c, is_alt_state::escape, depth) +: c.set_enabled && *p == c.set_open && +is_set(cx::next(p), pend, c, is_set_state::not_or_first) +? is_alt(set_end(cx::next(p), pend, c, set_end_state::not_or_first), +pend, c, state, depth) +: *p == c.alt_open +? is_alt(cx::next(p), pend, c, state, depth + 1) +: *p == c.alt_close +? depth == 1 || +is_alt(cx::next(p), pend, c, state, depth - 1) +: is_alt(cx::next(p), pend, c, state, depth) +: +state == is_alt_state::escape +? is_alt(cx::next(p), pend, c, is_alt_state::next, depth) +: throw std::logic_error( +"The program execution should never end up here throwing this " +"exception")); +#endif +} +enum class alt_end_state +{ +open, +next, +escape +}; +template +constexpr PatternIterator alt_end( +PatternIterator p, PatternIterator pend, +const cards>& c = cards>(), +alt_end_state state = alt_end_state::open, int depth = 0) +{ +#if cfg_HAS_CONSTEXPR14 +if (!c.alt_enabled) +{ +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::invalid_argument("The use of alternatives is disabled"); +#else +return throw_invalid_argument(p, "The use of alternatives is disabled"); +#endif +} +while (p != pend) +{ +switch (state) +{ +case alt_end_state::open: +if (*p != c.alt_open) +{ +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::invalid_argument("The given pattern is not a valid alternative"); +#else +return throw_invalid_argument(p, "The given pattern is not a valid alternative"); +#endif +} +state = alt_end_state::next; +++depth; +break; +case alt_end_state::next: +if (*p == c.escape) +{ +state = alt_end_state::escape; +} +else if (c.set_enabled && *p == c.set_open && +is_set(cx::next(p), pend, c, is_set_state::not_or_first)) +{ +p = cx::prev(set_end(cx::next(p), pend, c, set_end_state::not_or_first)); +} +else if (*p == c.alt_open) +{ +++depth; +} +else if (*p == c.alt_close) +{ +--depth; +if (depth == 0) +{ +return cx::next(p); +} +} +break; +case alt_end_state::escape: +state = alt_end_state::next; +break; +default: +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::logic_error( +"The program execution should never end up here throwing this exception"); +#else +return throw_logic_error( +p, "The program execution should never end up here throwing this exception"); +#endif +} +p = cx::next(p); +} +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::invalid_argument("The given pattern is not a valid alternative"); +#else +return throw_invalid_argument(p, "The given pattern is not a valid alternative"); +#endif +#else +return !c.alt_enabled +? throw std::invalid_argument("The use of alternatives is disabled") +: p == pend +? throw std::invalid_argument("The given pattern is not a valid alternative") +: state == alt_end_state::open +? *p == c.alt_open +? alt_end(cx::next(p), pend, c, alt_end_state::next, depth + 1) +: throw std::invalid_argument( +"The given pattern is not a valid alternative") +: state == alt_end_state::next +? *p == c.escape +? alt_end(cx::next(p), pend, c, alt_end_state::escape, depth) +: c.set_enabled && *p == c.set_open && +is_set(cx::next(p), pend, c, +is_set_state::not_or_first) +? alt_end(set_end(cx::next(p), pend, c, +set_end_state::not_or_first), +pend, c, state, depth) +: *p == c.alt_open +? alt_end(cx::next(p), pend, c, state, depth + 1) +: *p == c.alt_close +? depth == 1 ? cx::next(p) +: alt_end(cx::next(p), pend, c, +state, depth - 1) +: alt_end(cx::next(p), pend, c, state, depth) +: +state == alt_end_state::escape +? alt_end(cx::next(p), pend, c, alt_end_state::next, depth) +: throw std::logic_error( +"The program execution should never end up here throwing " +"this " +"exception"); +#endif +} +enum class alt_sub_end_state +{ +next, +escape +}; +template +constexpr PatternIterator alt_sub_end( +PatternIterator p, PatternIterator pend, +const cards>& c = cards>(), +alt_sub_end_state state = alt_sub_end_state::next, int depth = 1) +{ +#if cfg_HAS_CONSTEXPR14 +if (!c.alt_enabled) +{ +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::invalid_argument("The use of alternatives is disabled"); +#else +return throw_invalid_argument(p, "The use of alternatives is disabled"); +#endif +} +while (p != pend) +{ +switch (state) +{ +case alt_sub_end_state::next: +if (*p == c.escape) +{ +state = alt_sub_end_state::escape; +} +else if (c.set_enabled && *p == c.set_open && +is_set(cx::next(p), pend, c, is_set_state::not_or_first)) +{ +p = cx::prev(set_end(cx::next(p), pend, c, set_end_state::not_or_first)); +} +else if (*p == c.alt_open) +{ +++depth; +} +else if (*p == c.alt_close) +{ +--depth; +if (depth == 0) +{ +return p; +} +} +else if (*p == c.alt_or) +{ +if (depth == 1) +{ +return p; +} +} +break; +case alt_sub_end_state::escape: +state = alt_sub_end_state::next; +break; +default: +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::logic_error( +"The program execution should never end up here throwing this exception"); +#else +return throw_logic_error( +p, "The program execution should never end up here throwing this exception"); +#endif +} +p = cx::next(p); +} +#if cfg_HAS_FULL_FEATURED_CONSTEXPR14 +throw std::invalid_argument("The given pattern is not a valid alternative"); +#else +return throw_invalid_argument(p, "The given pattern is not a valid alternative"); +#endif +#else +return !c.alt_enabled +? throw std::invalid_argument("The use of alternatives is disabled") +: p == pend +? throw std::invalid_argument("The given pattern is not a valid alternative") +: state == alt_sub_end_state::next +? *p == c.escape +? alt_sub_end(cx::next(p), pend, c, alt_sub_end_state::escape, depth) +: c.set_enabled && *p == c.set_open && +is_set(cx::next(p), pend, c, is_set_state::not_or_first) +? alt_sub_end(set_end(cx::next(p), pend, c, +set_end_state::not_or_first), +pend, c, state, depth) +: *p == c.alt_open +? alt_sub_end(cx::next(p), pend, c, state, depth + 1) +: *p == c.alt_close +? depth == 1 ? p : alt_sub_end(cx::next(p), pend, +c, state, depth - 1) +: *p == c.alt_or +? depth == 1 ? p +: alt_sub_end(cx::next(p), pend, +c, state, depth) +: alt_sub_end(cx::next(p), pend, c, state, +depth) +: +state == alt_sub_end_state::escape +? alt_sub_end(cx::next(p), pend, c, alt_sub_end_state::next, depth) +: throw std::logic_error( +"The program execution should never end up here throwing " +"this " +"exception"); +#endif +} +template > +constexpr match_result match( +SequenceIterator s, SequenceIterator send, PatternIterator p, PatternIterator pend, +const cards>& c = cards>(), +const EqualTo& equal_to = EqualTo(), bool partial = false, bool escape = false); +template > +constexpr match_result match_alt( +SequenceIterator s, SequenceIterator send, PatternIterator p1, PatternIterator p1end, +PatternIterator p2, PatternIterator p2end, +const cards>& c = cards>(), +const EqualTo& equal_to = EqualTo(), bool partial = false) +{ +#if cfg_HAS_CONSTEXPR14 +auto result1 = match(s, send, p1, p1end, c, equal_to, true); +if (result1) +{ +auto result2 = match(result1.s, send, p2, p2end, c, equal_to, partial); +if (result2) +{ +return result2; +} +} +p1 = cx::next(p1end); +if (p1 == p2) +{ +return make_match_result(false, s, p1end); +} +return match_alt(s, send, p1, alt_sub_end(p1, p2, c), p2, p2end, c, equal_to, partial); +#else +return match(s, send, p1, p1end, c, equal_to, true) && +match(match(s, send, p1, p1end, c, equal_to, true).s, send, p2, p2end, c, equal_to, +partial) +? match(match(s, send, p1, p1end, c, equal_to, true).s, send, p2, p2end, c, equal_to, +partial) +: cx::next(p1end) == p2 +? make_match_result(false, s, p1end) +: match_alt(s, send, cx::next(p1end), alt_sub_end(cx::next(p1end), p2, c), p2, +p2end, c, equal_to, partial); +#endif +} +template +constexpr match_result match( +SequenceIterator s, SequenceIterator send, PatternIterator p, PatternIterator pend, +const cards>& c, const EqualTo& equal_to, bool partial, +bool escape) +{ +#if cfg_HAS_CONSTEXPR14 +if (p == pend) +{ +return make_match_result(partial || s == send, s, p); +} +if (escape) +{ +if (s == send || !equal_to(*s, *p)) +{ +return make_match_result(false, s, p); +} +return match(cx::next(s), send, cx::next(p), pend, c, equal_to, partial); +} +if (*p == c.anything) +{ +auto result = match(s, send, cx::next(p), pend, c, equal_to, partial); +if (result) +{ +return result; +} +if (s == send) +{ +return make_match_result(false, s, p); +} +return match(cx::next(s), send, p, pend, c, equal_to, partial); +} +if (*p == c.single) +{ +if (s == send) +{ +return make_match_result(false, s, p); +} +return match(cx::next(s), send, cx::next(p), pend, c, equal_to, partial); +} +if (*p == c.escape) +{ +return match(s, send, cx::next(p), pend, c, equal_to, partial, true); +} +if (c.set_enabled && *p == c.set_open && is_set(cx::next(p), pend, c, is_set_state::not_or_first)) +{ +auto result = +match_set(s, send, cx::next(p), pend, c, equal_to, match_set_state::not_or_first_in); +if (!result) +{ +return result; +} +return match(cx::next(s), send, set_end(cx::next(p), pend, c, set_end_state::not_or_first), +pend, c, equal_to, partial); +} +if (c.alt_enabled && *p == c.alt_open && is_alt(cx::next(p), pend, c, is_alt_state::next, 1)) +{ +auto p_alt_end = alt_end(cx::next(p), pend, c, alt_end_state::next, 1); +return match_alt(s, send, cx::next(p), alt_sub_end(cx::next(p), p_alt_end, c), p_alt_end, pend, +c, equal_to, partial); +} +if (s == send || !equal_to(*s, *p)) +{ +return make_match_result(false, s, p); +} +return match(cx::next(s), send, cx::next(p), pend, c, equal_to, partial); +#else +return p == pend +? make_match_result(partial || s == send, s, p) +: escape +? s == send || !equal_to(*s, *p) +? make_match_result(false, s, p) +: match(cx::next(s), send, cx::next(p), pend, c, equal_to, partial) +: *p == c.anything +? match(s, send, cx::next(p), pend, c, equal_to, partial) +? match(s, send, cx::next(p), pend, c, equal_to, partial) +: s == send ? make_match_result(false, s, p) +: match(cx::next(s), send, p, pend, c, equal_to, partial) +: *p == c.single +? s == send ? make_match_result(false, s, p) +: match(cx::next(s), send, cx::next(p), pend, c, +equal_to, partial) +: *p == c.escape +? match(s, send, cx::next(p), pend, c, equal_to, partial, true) +: c.set_enabled && *p == c.set_open && +is_set(cx::next(p), pend, c, +is_set_state::not_or_first) +? !match_set(s, send, cx::next(p), pend, c, equal_to, +match_set_state::not_or_first_in) +? match_set(s, send, cx::next(p), pend, c, +equal_to, +match_set_state::not_or_first_in) +: match(cx::next(s), send, +set_end(cx::next(p), pend, c, +set_end_state::not_or_first), +pend, c, equal_to, partial) +: c.alt_enabled && *p == c.alt_open && +is_alt(cx::next(p), pend, c, +is_alt_state::next, 1) +? match_alt( +s, send, cx::next(p), +alt_sub_end(cx::next(p), +alt_end(cx::next(p), pend, c, +alt_end_state::next, 1), +c), +alt_end(cx::next(p), pend, c, +alt_end_state::next, 1), +pend, c, equal_to, partial) +: s == send || !equal_to(*s, *p) +? make_match_result(false, s, p) +: match(cx::next(s), send, cx::next(p), pend, +c, equal_to, partial); +#endif +} +} +template > +constexpr full_match_result, const_iterator_t> match( +Sequence&& sequence, Pattern&& pattern, +const cards>& c = cards>(), +const EqualTo& equal_to = EqualTo()) +{ +return detail::make_full_match_result( +cx::cbegin(sequence), cx::cend(sequence), cx::cbegin(pattern), cx::cend(pattern), +detail::match(cx::cbegin(sequence), cx::cend(std::forward(sequence)), +cx::cbegin(pattern), cx::cend(std::forward(pattern)), c, equal_to)); +} +template , +typename = typename std::enable_if::value>::type> +constexpr full_match_result, const_iterator_t> match( +Sequence&& sequence, Pattern&& pattern, const EqualTo& equal_to) +{ +return match(std::forward(sequence), std::forward(pattern), +cards>(), equal_to); +} +} +#endif +#ifndef WILDCARDS_MATCHER_HPP +#define WILDCARDS_MATCHER_HPP +#include +#include +#include +#ifndef CX_STRING_VIEW_HPP +#define CX_STRING_VIEW_HPP +#include +#include +#ifndef CX_ALGORITHM_HPP +#define CX_ALGORITHM_HPP +namespace cx +{ +template +constexpr bool equal(Iterator1 first1, Iterator1 last1, Iterator2 first2, Iterator2 last2) +{ +#if cfg_HAS_CONSTEXPR14 +while (first1 != last1 && first2 != last2 && *first1 == *first2) +{ +++first1, ++first2; +} +return first1 == last1 && first2 == last2; +#else +return first1 != last1 && first2 != last2 && *first1 == *first2 +? equal(first1 + 1, last1, first2 + 1, last2) +: first1 == last1 && first2 == last2; +#endif +} +} +#endif +namespace cx +{ +template +class basic_string_view +{ +public: +using value_type = T; +constexpr basic_string_view() = default; +template +constexpr basic_string_view(const T (&str)[N]) : data_{&str[0]}, size_{N - 1} +{ +} +constexpr basic_string_view(const T* str, std::size_t s) : data_{str}, size_{s} +{ +} +constexpr const T* data() const +{ +return data_; +} +constexpr std::size_t size() const +{ +return size_; +} +constexpr bool empty() const +{ +return size() == 0; +} +constexpr const T* begin() const +{ +return data_; +} +constexpr const T* cbegin() const +{ +return begin(); +} +constexpr const T* end() const +{ +return data_ + size_; +} +constexpr const T* cend() const +{ +return end(); +} +private: +const T* data_{nullptr}; +std::size_t size_{0}; +}; +template +constexpr bool operator==(const basic_string_view& lhs, const basic_string_view& rhs) +{ +return equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); +} +template +constexpr bool operator!=(const basic_string_view& lhs, const basic_string_view& rhs) +{ +return !(lhs == rhs); +} +template +std::basic_ostream& operator<<(std::basic_ostream& o, const basic_string_view& s) +{ +o << s.data(); +return o; +} +template +constexpr basic_string_view make_string_view(const T (&str)[N]) +{ +return {str, N - 1}; +} +template +constexpr basic_string_view make_string_view(const T* str, std::size_t s) +{ +return {str, s}; +} +using string_view = basic_string_view; +using u16string_view = basic_string_view; +using u32string_view = basic_string_view; +using wstring_view = basic_string_view; +namespace literals +{ +constexpr string_view operator"" _sv(const char* str, std::size_t s) +{ +return {str, s}; +} +constexpr u16string_view operator"" _sv(const char16_t* str, std::size_t s) +{ +return {str, s}; +} +constexpr u32string_view operator"" _sv(const char32_t* str, std::size_t s) +{ +return {str, s}; +} +constexpr wstring_view operator"" _sv(const wchar_t* str, std::size_t s) +{ +return {str, s}; +} +} +} +#endif +namespace wildcards +{ +template > +class matcher +{ +public: +constexpr explicit matcher(Pattern&& pattern, const cards>& c = +cards>(), +const EqualTo& equal_to = EqualTo()) +: p_{cx::cbegin(pattern)}, +pend_{cx::cend(std::forward(pattern))}, +c_{c}, +equal_to_{equal_to} +{ +} +constexpr matcher(Pattern&& pattern, const EqualTo& equal_to) +: p_{cx::cbegin(pattern)}, +pend_{cx::cend(std::forward(pattern))}, +c_{cards>()}, +equal_to_{equal_to} +{ +} +template +constexpr full_match_result, const_iterator_t> matches( +Sequence&& sequence) const +{ +return detail::make_full_match_result( +cx::cbegin(sequence), cx::cend(sequence), p_, pend_, +detail::match(cx::cbegin(sequence), cx::cend(std::forward(sequence)), p_, pend_, +c_, equal_to_)); +} +private: +const_iterator_t p_; +const_iterator_t pend_; +cards> c_; +EqualTo equal_to_; +}; +template > +constexpr matcher make_matcher( +Pattern&& pattern, +const cards>& c = cards>(), +const EqualTo& equal_to = EqualTo()) +{ +return matcher{std::forward(pattern), c, equal_to}; +} +template , +typename = typename std::enable_if::value>::type> +constexpr matcher make_matcher(Pattern&& pattern, const EqualTo& equal_to) +{ +return make_matcher(std::forward(pattern), cards>(), equal_to); +} +namespace literals +{ +constexpr auto operator"" _wc(const char* str, std::size_t s) +-> decltype(make_matcher(cx::make_string_view(str, s + 1))) +{ +return make_matcher(cx::make_string_view(str, s + 1)); +} +constexpr auto operator"" _wc(const char16_t* str, std::size_t s) +-> decltype(make_matcher(cx::make_string_view(str, s + 1))) +{ +return make_matcher(cx::make_string_view(str, s + 1)); +} +constexpr auto operator"" _wc(const char32_t* str, std::size_t s) +-> decltype(make_matcher(cx::make_string_view(str, s + 1))) +{ +return make_matcher(cx::make_string_view(str, s + 1)); +} +constexpr auto operator"" _wc(const wchar_t* str, std::size_t s) +-> decltype(make_matcher(cx::make_string_view(str, s + 1))) +{ +return make_matcher(cx::make_string_view(str, s + 1)); +} +} +} +#endif +#endif diff --git a/source/common/fonts/singlelumpfont.cpp b/source/common/fonts/singlelumpfont.cpp index 0a8a099e4..0d5839401 100644 --- a/source/common/fonts/singlelumpfont.cpp +++ b/source/common/fonts/singlelumpfont.cpp @@ -125,7 +125,7 @@ FSingleLumpFont::FSingleLumpFont (const char *name, int lump) : FFont(lump) FontName = name; auto data1 = fileSystem.ReadFile (lump); - auto data = data1.GetBytes(); + auto data = data1.bytes(); if (data[0] == 0xE1 && data[1] == 0xE6 && data[2] == 0xD5 && data[3] == 0x1A) { @@ -475,7 +475,7 @@ void FSingleLumpFont::LoadBMF(int lump, const uint8_t *data) void FSingleLumpFont::CheckFON1Chars() { auto memLump = fileSystem.ReadFile(Lump); - auto data = memLump.GetBytes(); + auto data = memLump.bytes(); const uint8_t* data_p; data_p = data + 8; diff --git a/source/common/menu/savegamemanager.cpp b/source/common/menu/savegamemanager.cpp index 4682be7f2..3f2243a87 100644 --- a/source/common/menu/savegamemanager.cpp +++ b/source/common/menu/savegamemanager.cpp @@ -316,7 +316,7 @@ unsigned FSavegameManagerBase::ExtractSaveData(int index) auto pic = resf->FindEntry("savepic.png"); if (pic >= 0) { - FileReader picreader = resf->GetEntryReader(pic); + FileReader picreader = resf->GetEntryReader(pic, FileSys::READER_NEW, FileSys::READERFLAG_SEEKABLE); PNGHandle *png = M_VerifyPNG(picreader); if (png != nullptr) { diff --git a/source/common/models/model.cpp b/source/common/models/model.cpp index 4b9ab0b00..ce61bfe54 100644 --- a/source/common/models/model.cpp +++ b/source/common/models/model.cpp @@ -165,7 +165,7 @@ unsigned FindModel(const char * path, const char * modelfile, bool silent) int len = fileSystem.FileLength(lump); auto lumpd = fileSystem.ReadFile(lump); - const char * buffer = lumpd.GetString(); + const char * buffer = lumpd.string(); if ( (size_t)fullname.LastIndexOf("_d.3d") == fullname.Len()-5 ) { diff --git a/source/common/models/models_iqm.cpp b/source/common/models/models_iqm.cpp index d22c0d9d5..1184a9390 100644 --- a/source/common/models/models_iqm.cpp +++ b/source/common/models/models_iqm.cpp @@ -274,7 +274,7 @@ void IQMModel::LoadGeometry() try { auto lumpdata = fileSystem.ReadFile(mLumpNum); - IQMFileReader reader(lumpdata.GetMem(), (int)lumpdata.GetSize()); + IQMFileReader reader(lumpdata.data(), (int)lumpdata.size()); Vertices.Resize(NumVertices); for (IQMVertexArray& vertexArray : VertexArrays) diff --git a/source/common/models/models_md2.cpp b/source/common/models/models_md2.cpp index 3ba364c03..280c4d265 100644 --- a/source/common/models/models_md2.cpp +++ b/source/common/models/models_md2.cpp @@ -178,7 +178,7 @@ void FDMDModel::LoadGeometry() { static int axis[3] = { VX, VY, VZ }; auto lumpdata = fileSystem.ReadFile(mLumpNum); - auto buffer = lumpdata.GetString(); + auto buffer = lumpdata.string(); texCoords = new FTexCoord[info.numTexCoords]; memcpy(texCoords, buffer + info.offsetTexCoords, info.numTexCoords * sizeof(FTexCoord)); @@ -502,7 +502,7 @@ void FMD2Model::LoadGeometry() static int axis[3] = { VX, VY, VZ }; uint8_t *md2_frames; auto lumpdata = fileSystem.ReadFile(mLumpNum); - auto buffer = lumpdata.GetString(); + auto buffer = lumpdata.string(); texCoords = new FTexCoord[info.numTexCoords]; memcpy(texCoords, (uint8_t*)buffer + info.offsetTexCoords, info.numTexCoords * sizeof(FTexCoord)); diff --git a/source/common/models/models_md3.cpp b/source/common/models/models_md3.cpp index 5330611fc..327cbff88 100644 --- a/source/common/models/models_md3.cpp +++ b/source/common/models/models_md3.cpp @@ -190,7 +190,7 @@ bool FMD3Model::Load(const char * path, int lumpnum, const char * buffer, int le void FMD3Model::LoadGeometry() { auto lumpdata = fileSystem.ReadFile(mLumpNum); - auto buffer = lumpdata.GetString(); + auto buffer = lumpdata.string(); md3_header_t * hdr = (md3_header_t *)buffer; md3_surface_t * surf = (md3_surface_t*)(buffer + LittleLong(hdr->Ofs_Surfaces)); diff --git a/source/common/models/models_ue1.cpp b/source/common/models/models_ue1.cpp index c2f6929eb..810659261 100644 --- a/source/common/models/models_ue1.cpp +++ b/source/common/models/models_ue1.cpp @@ -71,9 +71,9 @@ void FUE1Model::LoadGeometry() { const char *buffer, *buffer2; auto lump = fileSystem.ReadFile(mDataLump); - buffer = lump.GetString(); + buffer = lump.string(); auto lump2 = fileSystem.ReadFile(mAnivLump); - buffer2 = lump2.GetString(); + buffer2 = lump2.string(); // map structures dhead = (const d3dhead*)(buffer); dpolys = (const d3dpoly*)(buffer+sizeof(d3dhead)); diff --git a/source/common/models/voxels.cpp b/source/common/models/voxels.cpp index e5b61bcb6..a2e371582 100644 --- a/source/common/models/voxels.cpp +++ b/source/common/models/voxels.cpp @@ -162,8 +162,8 @@ FVoxel *R_LoadKVX(int lumpnum) int i, j, n; auto lump = fileSystem.ReadFile(lumpnum); // FileData adds an extra 0 byte to the end. - auto rawvoxel = lump.GetBytes(); - int voxelsize = (int)(lump.GetSize()); + auto rawvoxel = lump.bytes(); + int voxelsize = (int)(lump.size()); // Oh, KVX, why couldn't you have a proper header? We'll just go through // and collect each MIP level, doing lots of range checking, and if the diff --git a/source/common/rendering/gl/gl_shaderprogram.cpp b/source/common/rendering/gl/gl_shaderprogram.cpp index d5e8dd321..1c4fd48a4 100644 --- a/source/common/rendering/gl/gl_shaderprogram.cpp +++ b/source/common/rendering/gl/gl_shaderprogram.cpp @@ -89,7 +89,6 @@ void FShaderProgram::Compile(ShaderType type, const char *lumpName, const char * { int lump = fileSystem.CheckNumForFullName(lumpName); if (lump == -1) I_FatalError("Unable to load '%s'", lumpName); - auto sp = fileSystem.ReadFile(lump); FString code = GetStringFromLump(lump); Compile(type, lumpName, code, defines, maxGlslVersion); diff --git a/source/common/rendering/gles/gles_shader.cpp b/source/common/rendering/gles/gles_shader.cpp index f037e3e4a..dba9a233a 100644 --- a/source/common/rendering/gles/gles_shader.cpp +++ b/source/common/rendering/gles/gles_shader.cpp @@ -383,11 +383,9 @@ bool FShader::Load(const char * name, const char * vert_prog_lump_, const char * int vp_lump = fileSystem.CheckNumForFullName(vert_prog_lump.GetChars(), 0); if (vp_lump == -1) I_Error("Unable to load '%s'", vert_prog_lump.GetChars()); - auto vp_data = fileSystem.ReadFile(vp_lump); int fp_lump = fileSystem.CheckNumForFullName(frag_prog_lump.GetChars(), 0); if (fp_lump == -1) I_Error("Unable to load '%s'", frag_prog_lump.GetChars()); - auto fp_data = fileSystem.ReadFile(fp_lump); @@ -422,7 +420,6 @@ bool FShader::Load(const char * name, const char * vert_prog_lump_, const char * { int pp_lump = fileSystem.CheckNumForFullName(proc_prog_lump.GetChars()); if (pp_lump == -1) I_Error("Unable to load '%s'", proc_prog_lump.GetChars()); - auto ppf = fileSystem.ReadFile(pp_lump); FString pp_data = GetStringFromLump(pp_lump); if (pp_data.IndexOf("ProcessMaterial") < 0 && pp_data.IndexOf("SetupMaterial") < 0) diff --git a/source/common/rendering/gles/gles_shaderprogram.cpp b/source/common/rendering/gles/gles_shaderprogram.cpp index a6f0453c2..efc94c71d 100644 --- a/source/common/rendering/gles/gles_shaderprogram.cpp +++ b/source/common/rendering/gles/gles_shaderprogram.cpp @@ -89,7 +89,6 @@ void FShaderProgram::Compile(ShaderType type, const char *lumpName, const char * { int lump = fileSystem.CheckNumForFullName(lumpName); if (lump == -1) I_FatalError("Unable to load '%s'", lumpName); - auto sp = fileSystem.ReadFile(lump); FString code = GetStringFromLump(lump); Compile(type, lumpName, code, defines, maxGlslVersion); } diff --git a/source/common/rendering/vulkan/shaders/vk_ppshader.cpp b/source/common/rendering/vulkan/shaders/vk_ppshader.cpp index dc4ace0f9..dcf72a06d 100644 --- a/source/common/rendering/vulkan/shaders/vk_ppshader.cpp +++ b/source/common/rendering/vulkan/shaders/vk_ppshader.cpp @@ -69,7 +69,6 @@ FString VkPPShader::LoadShaderCode(const FString &lumpName, const FString &defin { int lump = fileSystem.CheckNumForFullName(lumpName.GetChars()); if (lump == -1) I_FatalError("Unable to load '%s'", lumpName.GetChars()); - auto sp = fileSystem.ReadFile(lump); FString code = GetStringFromLump(lump); FString patchedCode; diff --git a/source/common/rendering/vulkan/shaders/vk_shader.cpp b/source/common/rendering/vulkan/shaders/vk_shader.cpp index 227b8cca6..618fa2ca9 100644 --- a/source/common/rendering/vulkan/shaders/vk_shader.cpp +++ b/source/common/rendering/vulkan/shaders/vk_shader.cpp @@ -475,7 +475,6 @@ FString VkShaderManager::LoadPrivateShaderLump(const char *lumpname) { int lump = fileSystem.CheckNumForFullName(lumpname, 0); if (lump == -1) I_Error("Unable to load '%s'", lumpname); - auto data = fileSystem.ReadFile(lump); return GetStringFromLump(lump); } diff --git a/source/common/textures/formats/anmtexture.cpp b/source/common/textures/formats/anmtexture.cpp index dbc51bae5..1eb7f4949 100644 --- a/source/common/textures/formats/anmtexture.cpp +++ b/source/common/textures/formats/anmtexture.cpp @@ -106,10 +106,10 @@ FAnmTexture::FAnmTexture (int lumpnum, int w, int h) void FAnmTexture::ReadFrame(uint8_t *pixels, uint8_t *palette) { auto lump = fileSystem.ReadFile (SourceLump); - auto source = lump.GetBytes(); + auto source = lump.bytes(); std::unique_ptr anim = std::make_unique(); // note that this struct is very large and should not go onto the stack! - if (ANIM_LoadAnim(anim.get(), source, (int)lump.GetSize()) >= 0) + if (ANIM_LoadAnim(anim.get(), source, (int)lump.size()) >= 0) { int numframes = ANIM_NumFrames(anim.get()); if (numframes >= 1) diff --git a/source/common/textures/formats/automaptexture.cpp b/source/common/textures/formats/automaptexture.cpp index a3b16c720..2ebdc4d53 100644 --- a/source/common/textures/formats/automaptexture.cpp +++ b/source/common/textures/formats/automaptexture.cpp @@ -93,7 +93,7 @@ PalettedPixels FAutomapTexture::CreatePalettedPixels(int conversion, int frame) { int x, y; auto data = fileSystem.ReadFile (SourceLump); - auto indata = data.GetBytes(); + auto indata = data.bytes(); PalettedPixels Pixels(Width * Height); diff --git a/source/common/textures/formats/imgztexture.cpp b/source/common/textures/formats/imgztexture.cpp index 831733a2b..9419c64a6 100644 --- a/source/common/textures/formats/imgztexture.cpp +++ b/source/common/textures/formats/imgztexture.cpp @@ -121,7 +121,7 @@ FIMGZTexture::FIMGZTexture (int lumpnum, uint16_t w, uint16_t h, int16_t l, int1 PalettedPixels FIMGZTexture::CreatePalettedPixels(int conversion, int frame) { auto lump = fileSystem.ReadFile (SourceLump); - auto imgz = (const ImageHeader *)lump.GetMem(); + auto imgz = (const ImageHeader *)lump.data(); const uint8_t *data = (const uint8_t *)&imgz[1]; uint8_t *dest_p; diff --git a/source/common/textures/formats/patchtexture.cpp b/source/common/textures/formats/patchtexture.cpp index 0b6266a58..d7e8f3fca 100644 --- a/source/common/textures/formats/patchtexture.cpp +++ b/source/common/textures/formats/patchtexture.cpp @@ -186,7 +186,7 @@ PalettedPixels FPatchTexture::CreatePalettedPixels(int conversion, int frame) int x; auto lump = fileSystem.ReadFile (SourceLump); - const patch_t *patch = (const patch_t *)lump.GetMem(); + const patch_t *patch = (const patch_t *)lump.data(); maxcol = (const column_t *)((const uint8_t *)patch + fileSystem.FileLength (SourceLump) - 3); @@ -296,7 +296,7 @@ void FPatchTexture::DetectBadPatches () // It must be 256 pixels tall, and all its columns must have exactly // one post, where each post has a supposed length of 0. auto lump = fileSystem.ReadFile (SourceLump); - const patch_t *realpatch = (patch_t *)lump.GetMem(); + const patch_t *realpatch = (patch_t *)lump.data(); const uint32_t *cofs = realpatch->columnofs; int x, x2 = LittleShort(realpatch->width); diff --git a/source/common/textures/formats/qoitexture.cpp b/source/common/textures/formats/qoitexture.cpp index 2df651756..07511aaf9 100644 --- a/source/common/textures/formats/qoitexture.cpp +++ b/source/common/textures/formats/qoitexture.cpp @@ -141,14 +141,14 @@ int FQOITexture::CopyPixels(FBitmap *bmp, int conversion, int frame) constexpr auto QOI_COLOR_HASH = [](PalEntry C) { return (C.r * 3 + C.g * 5 + C.b * 7 + C.a * 11); }; auto lump = fileSystem.ReadFile(SourceLump); - if (lump.GetSize() < 22) return 0; // error + if (lump.size() < 22) return 0; // error PalEntry index[64] = {}; PalEntry pe = 0xff000000; size_t p = 14, run = 0; - size_t chunks_len = lump.GetSize() - 8; - auto bytes = lump.GetBytes(); + size_t chunks_len = lump.size() - 8; + auto bytes = lump.bytes(); for (int h = 0; h < Height; h++) { diff --git a/source/common/textures/formats/rawpagetexture.cpp b/source/common/textures/formats/rawpagetexture.cpp index cca68dd07..52b3f3c53 100644 --- a/source/common/textures/formats/rawpagetexture.cpp +++ b/source/common/textures/formats/rawpagetexture.cpp @@ -183,7 +183,7 @@ FRawPageTexture::FRawPageTexture (int lumpnum) PalettedPixels FRawPageTexture::CreatePalettedPixels(int conversion, int frame) { auto lump = fileSystem.ReadFile (SourceLump); - auto source = lump.GetBytes(); + auto source = lump.bytes(); const uint8_t *source_p = source; uint8_t *dest_p; @@ -216,8 +216,8 @@ int FRawPageTexture::CopyPixels(FBitmap *bmp, int conversion, int frame) { auto lump = fileSystem.ReadFile(SourceLump); auto plump = fileSystem.ReadFile(mPaletteLump); - auto source = lump.GetBytes(); - auto psource = plump.GetBytes(); + auto source = lump.bytes(); + auto psource = plump.bytes(); PalEntry paldata[256]; for (auto & pe : paldata) { diff --git a/source/common/textures/formats/startuptexture.cpp b/source/common/textures/formats/startuptexture.cpp index 510061f7c..c062251d8 100644 --- a/source/common/textures/formats/startuptexture.cpp +++ b/source/common/textures/formats/startuptexture.cpp @@ -165,7 +165,7 @@ FStartupTexture::FStartupTexture (int lumpnum) bUseGamePalette = false; auto lump = fileSystem.ReadFile (SourceLump); - auto source = lump.GetBytes(); + auto source = lump.bytes(); // Initialize the bitmap palette. // the palette is static so that the notches can share it. @@ -234,7 +234,7 @@ void PlanarToChunky(T* dest, const uint8_t* src, const T* remap, int width, int PalettedPixels FStartupTexture::CreatePalettedPixels(int conversion, int frame) { auto lump = fileSystem.ReadFile (SourceLump); - auto source = lump.GetBytes(); + auto source = lump.bytes(); const uint8_t *remap = ImageHelpers::GetRemap(conversion == luminance); @@ -254,7 +254,7 @@ PalettedPixels FStartupTexture::CreatePalettedPixels(int conversion, int frame) int FStartupTexture::CopyPixels(FBitmap *bmp, int conversion, int frame) { auto lump = fileSystem.ReadFile (SourceLump); - auto source = lump.GetBytes(); + auto source = lump.bytes(); PlanarToChunky((uint32_t*)bmp->GetPixels(), source + 48, startuppalette32, Width, Height); return 0; } @@ -282,7 +282,7 @@ FNotchTexture::FNotchTexture (int lumpnum, int width, int height) PalettedPixels FNotchTexture::CreatePalettedPixels(int conversion, int frame) { auto lump = fileSystem.ReadFile (SourceLump); - auto source = lump.GetBytes(); + auto source = lump.bytes(); const uint8_t *remap = ImageHelpers::GetRemap(conversion == luminance); TArray Work(Width*Height, true); @@ -305,7 +305,7 @@ PalettedPixels FNotchTexture::CreatePalettedPixels(int conversion, int frame) int FNotchTexture::CopyPixels(FBitmap *bmp, int conversion, int frame) { auto lump = fileSystem.ReadFile (SourceLump); - auto source = lump.GetBytes(); + auto source = lump.bytes(); auto Work = (uint32_t*)bmp->GetPixels(); for(int i = 0; i < Width * Height / 2; i++) @@ -339,7 +339,7 @@ FStrifeStartupTexture::FStrifeStartupTexture (int lumpnum, int w, int h) PalettedPixels FStrifeStartupTexture::CreatePalettedPixels(int conversion, int frame) { auto lump = fileSystem.ReadFile (SourceLump); - auto source = lump.GetBytes(); + auto source = lump.bytes(); PalettedPixels Pixels(Width*Height); const uint8_t *remap = ImageHelpers::GetRemap(conversion == luminance); ImageHelpers::FlipNonSquareBlockRemap(Pixels.Data(), source, Width, Height, Width, remap); diff --git a/source/common/textures/formats/webptexture.cpp b/source/common/textures/formats/webptexture.cpp index e86ed4a11..2df759645 100644 --- a/source/common/textures/formats/webptexture.cpp +++ b/source/common/textures/formats/webptexture.cpp @@ -142,7 +142,7 @@ int FWebPTexture::CopyPixels(FBitmap *bmp, int conversion, int frame) config.output.u.RGBA.stride = bmp->GetPitch(); config.output.is_external_memory = 1; - (void)WebPDecode(bytes.GetBytes(), bytes.GetSize(), &config); + (void)WebPDecode(bytes.bytes(), bytes.size(), &config); return 0; } diff --git a/source/common/textures/multipatchtexturebuilder.cpp b/source/common/textures/multipatchtexturebuilder.cpp index 7f94437d2..f57b1f65b 100644 --- a/source/common/textures/multipatchtexturebuilder.cpp +++ b/source/common/textures/multipatchtexturebuilder.cpp @@ -398,12 +398,12 @@ void FMultipatchTextureBuilder::AddTexturesLumps(int lump1, int lump2, int patch if (lump1 >= 0) { auto texdir = fileSystem.ReadFile(lump1); - AddTexturesLump(texdir.GetMem(), fileSystem.FileLength(lump1), lump1, patcheslump, firstdup, true); + AddTexturesLump(texdir.data(), fileSystem.FileLength(lump1), lump1, patcheslump, firstdup, true); } if (lump2 >= 0) { auto texdir = fileSystem.ReadFile(lump2); - AddTexturesLump(texdir.GetMem(), fileSystem.FileLength(lump2), lump2, patcheslump, firstdup, false); + AddTexturesLump(texdir.data(), fileSystem.FileLength(lump2), lump2, patcheslump, firstdup, false); } } diff --git a/source/common/textures/texturemanager.cpp b/source/common/textures/texturemanager.cpp index 570e01315..10a3d3367 100644 --- a/source/common/textures/texturemanager.cpp +++ b/source/common/textures/texturemanager.cpp @@ -585,7 +585,7 @@ void FTextureManager::AddGroup(int wadnum, int ns, ETextureType usetype) } progressFunc(); } - else if (ns == ns_flats && fileSystem.GetFileFlags(firsttx) & LUMPF_MAYBEFLAT) + else if (ns == ns_flats && fileSystem.GetFileFlags(firsttx) & RESFF_MAYBEFLAT) { if (fileSystem.CheckNumForName(Name, ns) < firsttx) { @@ -971,7 +971,7 @@ void FTextureManager::AddTexturesForWad(int wadnum, FMultipatchTextureBuilder &b if (ns == ns_global) { // In Zips all graphics must be in a separate namespace. - if (fileSystem.GetFileFlags(i) & LUMPF_FULLPATH) continue; + if (fileSystem.GetFileFlags(i) & RESFF_FULLPATH) continue; // Ignore lumps with empty names. if (fileSystem.CheckFileName(i, "")) continue; @@ -1425,7 +1425,7 @@ int FTextureManager::GuesstimateNumTextures () break; default: - if (fileSystem.GetFileFlags(i) & LUMPF_MAYBEFLAT) numtex++; + if (fileSystem.GetFileFlags(i) & RESFF_MAYBEFLAT) numtex++; break; } diff --git a/source/common/utility/cmdlib.cpp b/source/common/utility/cmdlib.cpp index 9bb06fb38..5933a91d5 100644 --- a/source/common/utility/cmdlib.cpp +++ b/source/common/utility/cmdlib.cpp @@ -1045,7 +1045,7 @@ void uppercopy(char* to, const char* from) FString GetStringFromLump(int lump, bool zerotruncate) { auto fd = fileSystem.ReadFile(lump); - FString ScriptBuffer(fd.GetString(), fd.GetSize()); + FString ScriptBuffer(fd.string(), fd.size()); if (zerotruncate) ScriptBuffer.Truncate(strlen(ScriptBuffer.GetChars())); // this is necessary to properly truncate the generated string to not contain 0 bytes. return ScriptBuffer; } diff --git a/source/common/utility/palette.cpp b/source/common/utility/palette.cpp index c0fcd9f5d..ea046a220 100644 --- a/source/common/utility/palette.cpp +++ b/source/common/utility/palette.cpp @@ -681,8 +681,8 @@ FString V_GetColorStringByName(const char* name, FScriptPosition* sc) } auto rgbNames = fileSystem.ReadFile(rgblump); - rgb = rgbNames.GetString(); - rgbEnd = rgb + rgbNames.GetSize(); + rgb = rgbNames.string(); + rgbEnd = rgb + rgbNames.size(); step = 0; namelen = strlen(name); @@ -930,11 +930,11 @@ int ReadPalette(int lumpnum, uint8_t* buffer) return 0; } auto lump = fileSystem.ReadFile(lumpnum); - auto lumpmem = lump.GetBytes(); + auto lumpmem = lump.bytes(); memset(buffer, 0, 768); FileReader fr; - fr.OpenMemory(lumpmem, lump.GetSize()); + fr.OpenMemory(lumpmem, lump.size()); auto png = M_VerifyPNG(fr); if (png) { @@ -964,7 +964,7 @@ int ReadPalette(int lumpnum, uint8_t* buffer) { FScanner sc; - sc.OpenMem(fileSystem.GetFileFullName(lumpnum), (char*)lumpmem, int(lump.GetSize())); + sc.OpenMem(fileSystem.GetFileFullName(lumpnum), (char*)lumpmem, int(lump.size())); sc.MustGetString(); sc.MustGetNumber(); // version - ignore sc.MustGetNumber(); @@ -982,7 +982,7 @@ int ReadPalette(int lumpnum, uint8_t* buffer) } else { - memcpy(buffer, lumpmem, min(768, lump.GetSize())); + memcpy(buffer, lumpmem, min(768, lump.size())); return 256; } } diff --git a/source/common/utility/writezip.cpp b/source/common/utility/writezip.cpp new file mode 100644 index 000000000..5c0b0e65b --- /dev/null +++ b/source/common/utility/writezip.cpp @@ -0,0 +1,239 @@ +/* +** writezip.cpp +** +**--------------------------------------------------------------------------- +** Copyright 1998-2009 Randy Heit +** Copyright 2005-2023 Christoph Oelckers +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** 3. The name of the author may not be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**--------------------------------------------------------------------------- +** +** +*/ + +#include +#include +#include "tarray.h" +#include "files.h" +#include "m_swap.h" +#include "w_zip.h" + +using FileSys::FCompressedBuffer; + + + +//========================================================================== +// +// time_to_dos +// +// Converts time from struct tm to the DOS format used by zip files. +// +//========================================================================== + +static std::pair time_to_dos(struct tm *time) +{ + std::pair val; + if (time == NULL || time->tm_year < 80) + { + val.first = val.second = 0; + } + else + { + val.first = (time->tm_year - 80) * 512 + (time->tm_mon + 1) * 32 + time->tm_mday; + val.second= time->tm_hour * 2048 + time->tm_min * 32 + time->tm_sec / 2; + } + return val; +} + +//========================================================================== +// +// append_to_zip +// +// Write a given file to the zipFile. +// +// zipfile: zip object to be written to +// +// returns: position = success, -1 = error +// +//========================================================================== + +static int AppendToZip(FileWriter *zip_file, const FCompressedBuffer &content, std::pair &dostime) +{ + FZipLocalFileHeader local; + int position; + + int flags = 0; + int method = content.mMethod; + if (method >= FileSys::METHOD_IMPLODE_MIN && method <= FileSys::METHOD_IMPLODE_MAX) + { + flags = method - FileSys::METHOD_IMPLODE_MIN; + method = FileSys::METHOD_IMPLODE; + } + else if (method == FileSys::METHOD_DEFLATE) + { + flags = 2; + } + else if (method >= 1337) + return -1; + + local.Magic = ZIP_LOCALFILE; + local.VersionToExtract[0] = 20; + local.VersionToExtract[1] = 0; + local.Flags = LittleShort((uint16_t)flags); + local.Method = LittleShort((uint16_t)method); + local.ModDate = LittleShort(dostime.first); + local.ModTime = LittleShort(dostime.second); + local.CRC32 = content.mCRC32; + local.UncompressedSize = LittleLong((unsigned)content.mSize); + local.CompressedSize = LittleLong((unsigned)content.mCompressedSize); + local.NameLength = LittleShort((unsigned short)strlen(content.filename)); + local.ExtraLength = 0; + + // Fill in local directory header. + + position = (int)zip_file->Tell(); + + // Write out the header, file name, and file data. + if (zip_file->Write(&local, sizeof(local)) != sizeof(local) || + zip_file->Write(content.filename, strlen(content.filename)) != strlen(content.filename) || + zip_file->Write(content.mBuffer, content.mCompressedSize) != content.mCompressedSize) + { + return -1; + } + return position; +} + + +//========================================================================== +// +// write_central_dir +// +// Writes the central directory entry for a file. +// +//========================================================================== + +int AppendCentralDirectory(FileWriter *zip_file, const FCompressedBuffer &content, std::pair &dostime, int position) +{ + FZipCentralDirectoryInfo dir; + + int flags = 0; + int method = content.mMethod; + if (method >= FileSys::METHOD_IMPLODE_MIN && method <= FileSys::METHOD_IMPLODE_MAX) + { + flags = method - FileSys::METHOD_IMPLODE_MIN; + method = FileSys::METHOD_IMPLODE; + } + else if (method == FileSys::METHOD_DEFLATE) + { + flags = 2; + } + else if (method >= 1337) + return -1; + + dir.Magic = ZIP_CENTRALFILE; + dir.VersionMadeBy[0] = 20; + dir.VersionMadeBy[1] = 0; + dir.VersionToExtract[0] = 20; + dir.VersionToExtract[1] = 0; + dir.Flags = LittleShort((uint16_t)flags); + dir.Method = LittleShort((uint16_t)method); + dir.ModTime = LittleShort(dostime.first); + dir.ModDate = LittleShort(dostime.second); + dir.CRC32 = content.mCRC32; + dir.CompressedSize32 = LittleLong((unsigned)content.mCompressedSize); + dir.UncompressedSize32 = LittleLong((unsigned)content.mSize); + dir.NameLength = LittleShort((unsigned short)strlen(content.filename)); + dir.ExtraLength = 0; + dir.CommentLength = 0; + dir.StartingDiskNumber = 0; + dir.InternalAttributes = 0; + dir.ExternalAttributes = 0; + dir.LocalHeaderOffset32 = LittleLong((unsigned)position); + + if (zip_file->Write(&dir, sizeof(dir)) != sizeof(dir) || + zip_file->Write(content.filename, strlen(content.filename)) != strlen(content.filename)) + { + return -1; + } + return 0; +} + +bool WriteZip(const char* filename, const FCompressedBuffer* content, size_t contentcount) +{ + // try to determine local time + struct tm *ltime; + time_t ttime; + ttime = time(nullptr); + ltime = localtime(&ttime); + auto dostime = time_to_dos(ltime); + + TArray positions; + + auto f = FileWriter::Open(filename); + if (f != nullptr) + { + for (size_t i = 0; i < contentcount; i++) + { + int pos = AppendToZip(f, content[i], dostime); + if (pos == -1) + { + delete f; + remove(filename); + return false; + } + positions.Push(pos); + } + + int dirofs = (int)f->Tell(); + for (size_t i = 0; i < contentcount; i++) + { + if (AppendCentralDirectory(f, content[i], dostime, positions[i]) < 0) + { + delete f; + remove(filename); + return false; + } + } + + // Write the directory terminator. + FZipEndOfCentralDirectory dirend; + dirend.Magic = ZIP_ENDOFDIR; + dirend.DiskNumber = 0; + dirend.FirstDisk = 0; + dirend.NumEntriesOnAllDisks = dirend.NumEntries = LittleShort((uint16_t)contentcount); + dirend.DirectoryOffset = LittleLong((unsigned)dirofs); + dirend.DirectorySize = LittleLong((uint32_t)(f->Tell() - dirofs)); + dirend.ZipCommentLength = 0; + if (f->Write(&dirend, sizeof(dirend)) != sizeof(dirend)) + { + delete f; + remove(filename); + return false; + } + delete f; + return true; + } + return false; +} diff --git a/source/core/initfs.cpp b/source/core/initfs.cpp index 15d52da0e..7254f0e17 100644 --- a/source/core/initfs.cpp +++ b/source/core/initfs.cpp @@ -433,10 +433,9 @@ void InitFileSystem(TArray& groups) lfi.embeddings = { "blood.rff", "sounds.rff" }; } - lfi.dotFilter = LumpFilter.GetChars(); - if (isDukeEngine()) lfi.gameTypeFilter.push_back("DukeEngine"); if (isDukeLike()) lfi.gameTypeFilter.push_back("DukeLike"); + lfi.gameTypeFilter.push_back(LumpFilter.GetChars()); lfi.postprocessFunc = [&]() { @@ -448,8 +447,8 @@ void InitFileSystem(TArray& groups) FILE* f = fopen("filesystem.dir", "wb"); for (int num = 0; num < fileSystem.GetNumEntries(); num++) { - auto fd = fileSystem.FileLength(num); - fprintf(f, "%.50s %60s %d\n", fileSystem.GetFileFullName(num), fileSystem.GetResourceFileFullName(fileSystem.GetFileContainer(num)), fd); + int64_t fd = fileSystem.FileLength(num); + fprintf(f, "%.50s %60s %lld\n", fileSystem.GetFileFullName(num), fileSystem.GetResourceFileFullName(fileSystem.GetFileContainer(num)), fd); } fclose(f); } diff --git a/source/core/savegamehelp.cpp b/source/core/savegamehelp.cpp index 8c80a4da5..7f1601258 100644 --- a/source/core/savegamehelp.cpp +++ b/source/core/savegamehelp.cpp @@ -69,6 +69,7 @@ #include "buildtiles.h" #include "fs_findfile.h" +#include "resourcefile.h" @@ -79,6 +80,7 @@ extern FString BackupSaveGame; int SaveVersion; void SerializeMap(FSerializer &arc); +bool WriteZip(const char* filename, const FileSys::FCompressedBuffer* content, size_t contentcount); BEGIN_BLD_NS FSerializer& Serialize(FSerializer& arc, const char* keyname, XWALL& w, XWALL* def); @@ -132,7 +134,7 @@ static void SerializeSession(FSerializer& arc) bool ReadSavegame(const char* name) { - auto savereader = FResourceFile::OpenResourceFile(name, true); + auto savereader = FileSys::FResourceFile::OpenResourceFile(name, true); if (savereader != nullptr) { @@ -245,7 +247,7 @@ bool WriteSavegame(const char* filename, const char *name) M_FinishPNG(&savepic); auto picdata = savepic.GetBuffer(); - FileSys::FCompressedBuffer bufpng = { picdata->size(), picdata->size(), FileSys::METHOD_STORED, 0, static_cast(crc32(0, &(*picdata)[0], picdata->size())), (char*)&(*picdata)[0] }; + FileSys::FCompressedBuffer bufpng = { picdata->size(), picdata->size(), FileSys::METHOD_STORED, static_cast(crc32(0, &(*picdata)[0], picdata->size())), (char*)&(*picdata)[0] }; TArray savegame_content; TArray savegame_filenames; diff --git a/source/games/blood/src/blood.cpp b/source/games/blood/src/blood.cpp index 4cdab30c4..15ab65bb2 100644 --- a/source/games/blood/src/blood.cpp +++ b/source/games/blood/src/blood.cpp @@ -560,8 +560,8 @@ void GameInterface::loadPalette(void) for (int i = 0; i < 5; i++) { auto pal = fileSystem.ReadFileFullName(PAL[i]); - if (pal.GetSize() < 768) I_FatalError("%s: file too small", PAL[i]); - paletteSetColorTable(i, (const uint8_t*)pal.GetMem(), false, false); + if (pal.size() < 768) I_FatalError("%s: file too small", PAL[i]); + paletteSetColorTable(i, pal.bytes(), false, false); } numshades = 64; @@ -574,12 +574,12 @@ void GameInterface::loadPalette(void) else continue; } auto data = fileSystem.ReadFile(lump); - if (data.GetSize() != 64 * 256) + if (data.size() != 64 * 256) { if (i < 15) I_FatalError("%s: Incorrect PLU size", PLU[i]); else continue; } - lookups.setTable(i, (const uint8_t*)data.GetMem()); + lookups.setTable(i, data.bytes()); } lookups.setFadeColor(1, 255, 255, 255); diff --git a/source/games/blood/src/sound.cpp b/source/games/blood/src/sound.cpp index bd77fae7c..dee1448aa 100644 --- a/source/games/blood/src/sound.cpp +++ b/source/games/blood/src/sound.cpp @@ -76,7 +76,7 @@ static void S_AddBloodSFX(int lumpnum) } auto sfxlump = fileSystem.ReadFile(lumpnum); - SFX* sfx = (SFX*)sfxlump.GetMem(); + SFX* sfx = (SFX*)sfxlump.data(); ByteSwapSFX(sfx); FStringf rawname("%s.raw", sfx->rawName); diff --git a/source/games/exhumed/src/sound.cpp b/source/games/exhumed/src/sound.cpp index 8ed78be35..f61c7dff6 100644 --- a/source/games/exhumed/src/sound.cpp +++ b/source/games/exhumed/src/sound.cpp @@ -177,7 +177,7 @@ int LoadSound(const char* name) { auto check = fileSystem.ReadFile(lump); bool loops = false; - if (check.GetSize() > 26 && check.GetString()[26] == 6 && !memcmp("Creative Voice File", check.GetString(), 19)) + if (check.size() > 26 && check.string()[26] == 6 && !memcmp("Creative Voice File", check.string(), 19)) { // This game uses the actual loop point information in the sound data as its only means to check if a sound is looped. loops = true; diff --git a/source/games/sw/src/scrip2.cpp b/source/games/sw/src/scrip2.cpp index 61089d893..809379407 100644 --- a/source/games/sw/src/scrip2.cpp +++ b/source/games/sw/src/scrip2.cpp @@ -78,7 +78,7 @@ TArray LoadScriptFile(const char *filename) return ret; } - ret.Resize(fp.GetLength() + 1); + ret.Resize((unsigned)fp.GetLength() + 1); if (fp.Read(ret.Data(), fp.GetLength()) < fp.GetLength()) { scriptline = 1;