zdoom/raze
0
0
Fork 0
mirror of https://github.com/ZDoom/Raze.git synced 2025-04-04 06:55:48 +00:00
Code Issues Projects Releases Packages Wiki Activity

filesystem update from GZDoom.

Browse source
This commit is contained in:
Christoph Oelckers 2023-12-17 12:48:03 +01:00
parent 84e10beab4
commit 36930d44bd
76 changed files with 3693 additions and 1914 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
source/CMakeLists.txt
View file

@ -1122,6 +1122,7 @@ set (PCH_SOURCES
common/utility/s_playlist.cpp
common/utility/name.cpp
common/utility/r_memory.cpp
common/utility/writezip.cpp
common/thirdparty/base64.cpp
common/thirdparty/md5.cpp
common/thirdparty/superfasthash.cpp
@ -1279,6 +1280,8 @@ set( GAME_SOURCES
common/filesystem/source/file_pak.cpp
common/filesystem/source/file_whres.cpp
common/filesystem/source/file_ssi.cpp
common/filesystem/source/file_hog.cpp
common/filesystem/source/file_mvl.cpp
common/filesystem/source/file_directory.cpp
common/filesystem/source/resourcefile.cpp
common/filesystem/source/files.cpp
@ -1286,6 +1289,7 @@ set( GAME_SOURCES
common/filesystem/source/fs_findfile.cpp
common/filesystem/source/fs_stringpool.cpp
common/filesystem/source/unicode.cpp
common/filesystem/source/critsec.cpp
)

8
source/common/audio/music/i_music.cpp
View file

@ -182,8 +182,8 @@ static void SetupGenMidi()
}
auto genmidi = fileSystem.ReadFile(lump);
if (genmidi.GetSize() < 8 + 175 * 36 || memcmp(genmidi.GetMem(), "#OPL_II#", 8)) return;
ZMusic_SetGenMidi(genmidi.GetBytes() + 8);
if (genmidi.size() < 8 + 175 * 36 || memcmp(genmidi.data(), "#OPL_II#", 8)) return;
ZMusic_SetGenMidi(genmidi.bytes() + 8);
}
static void SetupWgOpn()
@ -194,7 +194,7 @@ static void SetupWgOpn()
return;
}
auto data = fileSystem.ReadFile(lump);
ZMusic_SetWgOpn(data.GetMem(), (uint32_t)data.GetSize());
ZMusic_SetWgOpn(data.data(), (uint32_t)data.size());
}
static void SetupDMXGUS()
@ -206,7 +206,7 @@ static void SetupDMXGUS()
return;
}
auto data = fileSystem.ReadFile(lump);
ZMusic_SetDmxGus(data.GetMem(), (uint32_t)data.GetSize());
ZMusic_SetDmxGus(data.data(), (uint32_t)data.size());
}
#endif

2
source/common/audio/music/i_soundfont.cpp
View file

@ -218,7 +218,7 @@ FileReader FZipPatReader::OpenFile(const char *name)
auto lump = resf->FindEntry(name);
if (lump >= 0)
{
return resf->GetEntryReader(lump);
return resf->GetEntryReader(lump, FileSys::READER_NEW, FileSys::READERFLAG_SEEKABLE);
}
}
fr.OpenFile(name);

2
source/common/audio/music/music.cpp
View file

@ -703,6 +703,8 @@ bool S_ChangeMusic(const char* musicname, int order, bool looping, bool force)
// opening the music must be done by the game because it's different depending on the game's file system use.
FileReader reader = mus_cb.OpenMusic(musicname);
if (!reader.isOpen()) return false;
auto m = reader.Read();
reader.Seek(0, FileReader::SeekSet);
// shutdown old music
S_StopMusic(true);

57
source/common/console/c_cvars.cpp
View file

@ -44,6 +44,7 @@
#include "printf.h"
#include "palutil.h"
#include "i_interface.h"
#include "gstrings.h"
#include "dobject.h"
#include "dobjtype.h"
@ -1695,16 +1696,37 @@ CCMD (toggle)
}
}
void FBaseCVar::ListVars (const char *filter, bool plain)
void FBaseCVar::ListVars (const char *filter, int listtype)
{
int count = 0;
bool plain = listtype == LCT_Plain;
bool includedesc = listtype == LCT_FullSearch;
decltype(cvarMap)::Iterator it(cvarMap);
decltype(cvarMap)::Pair *pair;
while (it.NextPair(pair))
{
auto var = pair->Value;
if (CheckWildcards (filter, var->GetName()))
bool ismatch;
if (filter && includedesc)
{
// search always allow partial matches
// also allow matching to cvar name, localised description, and description language-id
FString SearchString = FString("*") + filter + "*";
ismatch = CheckWildcards (SearchString.GetChars(), var->GetName()) ||
CheckWildcards (SearchString.GetChars(), var->GetDescription().GetChars()) ||
CheckWildcards (SearchString.GetChars(), GStrings.localize(var->GetDescription().GetChars()));
}
else
{
ismatch = CheckWildcards (filter, var->GetName());
}
if (ismatch)
{
uint32_t flags = var->GetFlags();
if (plain)
@ -1718,7 +1740,8 @@ void FBaseCVar::ListVars (const char *filter, bool plain)
else
{
++count;
Printf ("%c%c%c%c%c %s = %s\n",
Printf ("%c%c%c%c%c %s = %s",
flags & CVAR_ARCHIVE ? 'A' : ' ',
flags & CVAR_USERINFO ? 'U' :
flags & CVAR_SERVERINFO ? 'S' :
@ -1730,6 +1753,16 @@ void FBaseCVar::ListVars (const char *filter, bool plain)
flags & CVAR_IGNORE ? 'X' : ' ',
var->GetName(),
var->GetHumanString());
if (includedesc)
if (var->GetDescription().Len())
Printf(" // \"%s\"\n", GStrings.localize(var->GetDescription().GetChars()));
else
Printf("\n");
else
Printf("\n");
}
}
}
@ -1740,17 +1773,29 @@ CCMD (cvarlist)
{
if (argv.argc() == 1)
{
FBaseCVar::ListVars (NULL, false);
FBaseCVar::ListVars (NULL, LCT_Default);
}
else
{
FBaseCVar::ListVars (argv[1], false);
FBaseCVar::ListVars (argv[1], LCT_Default);
}
}
CCMD (cvarlistplain)
{
FBaseCVar::ListVars (NULL, true);
FBaseCVar::ListVars (NULL, LCT_Plain);
}
CCMD (cvarsearch)
{
if (argv.argc() == 1)
{
FBaseCVar::ListVars (NULL, LCT_FullSearch);
}
else
{
FBaseCVar::ListVars (argv[1], LCT_FullSearch);
}
}
CCMD (archivecvar)

9
source/common/console/c_cvars.h
View file

@ -89,6 +89,13 @@ enum ECVarType
CVAR_Dummy, // Unknown
};
enum ListCCMDType
{
LCT_Default,
LCT_Plain,
LCT_FullSearch,
};
class FIntCVarRef;
union UCVarValue
@ -201,7 +208,7 @@ public:
static void MarkZSCallbacks ();
static void ResetColors (); // recalc color cvars' indices after screen change
static void ListVars (const char *filter, bool plain);
static void ListVars (const char *filter, int listtype);
const FString &GetDescription() const { return Description; };
const FString& GetToggleMessage(int which) { return ToggleMessages[which]; }

12
source/common/cutscenes/movieplayer.cpp
View file

@ -153,7 +153,7 @@ class AnmPlayer : public MoviePlayer
{
// This doesn't need its own class type
anim_t anim;
FileSys::ResourceData buffer;
FileSys::FileData buffer;
int numframes = 0;
int curframe = 1;
int frametime = 0;
@ -515,7 +515,7 @@ public:
}
else if (soundtrack >= 0)
{
FileReader reader = fileSystem.OpenFileReader(soundtrack);
FileReader reader = fileSystem.ReopenFileReader(soundtrack);
if (reader.isOpen())
{
MusicStream = ZMusic_OpenSong(GetMusicReader(reader), MDEV_DEFAULT, nullptr);
@ -794,7 +794,7 @@ public:
if (sound == INVALID_SOUND)
soundEngine->StopAllChannels();
else
soundEngine->StartSound(SOURCE_None, nullptr, nullptr, CHAN_AUTO, nostopsound ? CHANF_UI | CHANF_FORCE : CHANF_FORCE, sound, 1.f, ATTN_NONE);
soundEngine->StartSound(SOURCE_None, nullptr, nullptr, CHAN_AUTO, nostopsound ? CHANF_UI : CHANF_NONE, sound, 1.f, ATTN_NONE);
}
}
}
@ -838,10 +838,10 @@ MoviePlayer* OpenMovie(const char* filename, TArray<int>& ans, const int* framet
{
auto fn = StripExtension(filename);
DefaultExtension(fn, ".ivf");
fr = fileSystem.OpenFileReader(fn.GetChars());
fr = fileSystem.ReopenFileReader(fn.GetChars());
}
if (!fr.isOpen()) fr = fileSystem.OpenFileReader(filename);
if (!fr.isOpen()) fr = fileSystem.ReopenFileReader(filename);
if (!fr.isOpen())
{
size_t nLen = strlen(filename);
@ -849,7 +849,7 @@ MoviePlayer* OpenMovie(const char* filename, TArray<int>& ans, const int* framet
if (nLen >= 3 && isalpha(filename[0]) && filename[1] == ':' && filename[2] == '/')
{
filename += 3;
fr = fileSystem.OpenFileReader(filename);
fr = fileSystem.ReopenFileReader(filename);
}
if (!fr.isOpen())
{

8
source/common/engine/sc_man.cpp
View file

@ -200,10 +200,10 @@ void FScanner :: OpenLumpNum (int lump)
{
Close ();
{
auto mem = fileSystem.OpenFileReader(lump);
auto buff = ScriptBuffer.LockNewBuffer(mem.GetLength());
mem.Read(buff, mem.GetLength());
buff[mem.GetLength()] = 0;
auto len = fileSystem.FileLength(lump);
auto buff = ScriptBuffer.LockNewBuffer(len);
fileSystem.ReadFile(lump, buff);
buff[len] = 0;
ScriptBuffer.UnlockBuffer();
}
ScriptName = fileSystem.GetFileFullPath(lump).c_str();

1
source/common/engine/serializer.cpp
View file

@ -753,7 +753,6 @@ FCompressedBuffer FSerializer::GetCompressedOutput()
EndObject();
buff.filename = nullptr;
buff.mSize = (unsigned)w->mOutString.GetSize();
buff.mZipFlags = 0;
buff.mCRC32 = crc32(0, (const Bytef*)w->mOutString.GetString(), buff.mSize);
uint8_t *compressbuf = new uint8_t[buff.mSize+1];

6
source/common/engine/stringtable.cpp
View file

@ -63,8 +63,8 @@ void FStringTable::LoadStrings (const char *language)
{
auto lumpdata = fileSystem.ReadFile(lump);
if (!ParseLanguageCSV(lump, lumpdata.GetString(), lumpdata.GetSize()))
LoadLanguage (lump, lumpdata.GetString(), lumpdata.GetSize());
if (!ParseLanguageCSV(lump, lumpdata.string(), lumpdata.size()))
LoadLanguage (lump, lumpdata.string(), lumpdata.size());
}
UpdateLanguage(language);
allMacros.Clear();
@ -160,7 +160,7 @@ TArray<TArray<FString>> FStringTable::parseCSV(const char* buffer, size_t size)
bool FStringTable::readMacros(int lumpnum)
{
auto lumpdata = fileSystem.ReadFile(lumpnum);
auto data = parseCSV(lumpdata.GetString(), lumpdata.GetSize());
auto data = parseCSV(lumpdata.string(), lumpdata.size());
for (unsigned i = 1; i < data.Size(); i++)
{

63
source/common/filesystem/include/fs_decompress.h Normal file
View file

@ -0,0 +1,63 @@
#pragma once
#include "fs_files.h"
namespace FileSys {
// Zip compression methods, extended by some internal types to be passed to OpenDecompressor
enum ECompressionMethod
{
METHOD_STORED = 0,
METHOD_SHRINK = 1,
METHOD_IMPLODE = 6,
METHOD_DEFLATE = 8,
METHOD_BZIP2 = 12,
METHOD_LZMA = 14,
METHOD_XZ = 95,
METHOD_PPMD = 98,
METHOD_LZSS = 1337, // not used in Zips - this is for Console Doom compression
METHOD_ZLIB = 1338, // Zlib stream with header, used by compressed nodes.
METHOD_RFFCRYPT = 1339, // not actual compression but can be put in here to make handling easier.
METHOD_IMPLODE_MIN = 1000, // having discrete types for these avoids keeping around the GPFlags word in Zips.
METHOD_IMPLODE_0 = 1000,
METHOD_IMPLODE_2 = 1002,
METHOD_IMPLODE_4 = 1004,
METHOD_IMPLODE_6 = 1006,
METHOD_IMPLODE_MAX = 1006,
METHOD_INVALID = 0x7fff,
METHOD_TRANSFEROWNER = 0x8000,
};
enum EDecompressFlags
{
DCF_TRANSFEROWNER = 1,
DCF_SEEKABLE = 2,
DCF_EXCEPTIONS = 4,
DCF_CACHED = 8,
};
bool OpenDecompressor(FileReader& self, FileReader &parent, FileReader::Size length, int method, int flags = 0); // creates a decompressor stream. 'seekable' uses a buffered version so that the Seek and Tell methods can be used.
// This holds a compresed Zip entry with all needed info to decompress it.
struct FCompressedBuffer
{
size_t mSize;
size_t mCompressedSize;
int mMethod;
unsigned mCRC32;
char* mBuffer;
const char* filename;
bool Decompress(char* destbuffer);
void Clean()
{
mSize = mCompressedSize = 0;
if (mBuffer != nullptr)
{
delete[] mBuffer;
mBuffer = nullptr;
}
}
};
}

90
source/common/filesystem/include/fs_files.h
View file

@ -40,12 +40,11 @@
#include <stdio.h>
#include <stdint.h>
#include <stdarg.h>
#include <string.h>
#include <functional>
#include <vector>
#include "fs_swap.h"
#include "tarray.h"
namespace FileSys {
class FileSystemException : public std::exception
@ -72,26 +71,10 @@ public:
}
};
// Zip compression methods, extended by some internal types to be passed to OpenDecompressor
enum
{
METHOD_STORED = 0,
METHOD_SHRINK = 1,
METHOD_IMPLODE = 6,
METHOD_DEFLATE = 8,
METHOD_BZIP2 = 12,
METHOD_LZMA = 14,
METHOD_XZ = 95,
METHOD_PPMD = 98,
METHOD_LZSS = 1337, // not used in Zips - this is for Console Doom compression
METHOD_ZLIB = 1338, // Zlib stream with header, used by compressed nodes.
METHOD_TRANSFEROWNER = 0x8000,
};
class FileReader;
// an opaque memory buffer to the file's content. Can either own the memory or just point to an external buffer.
class ResourceData
class FileData
{
void* memory;
size_t length;
@ -99,13 +82,29 @@ class ResourceData
public:
using value_type = uint8_t;
ResourceData() { memory = nullptr; length = 0; owned = true; }
FileData() { memory = nullptr; length = 0; owned = true; }
FileData(const void* memory_, size_t len, bool own = true)
{
length = len;
if (own)
{
length = len;
memory = allocate(len);
if (memory_) memcpy(memory, memory_, len);
}
else
{
memory = (void*)memory_;
owned = false;
}
}
uint8_t* writable() const { return owned? (uint8_t*)memory : nullptr; }
const void* data() const { return memory; }
size_t size() const { return length; }
const char* string() const { return (const char*)memory; }
const uint8_t* bytes() const { return (const uint8_t*)memory; }
ResourceData& operator = (const ResourceData& copy)
FileData& operator = (const FileData& copy)
{
if (owned && memory) free(memory);
length = copy.length;
@ -119,7 +118,7 @@ public:
return *this;
}
ResourceData& operator = (ResourceData&& copy) noexcept
FileData& operator = (FileData&& copy) noexcept
{
if (owned && memory) free(memory);
length = copy.length;
@ -131,13 +130,13 @@ public:
return *this;
}
ResourceData(const ResourceData& copy)
FileData(const FileData& copy)
{
memory = nullptr;
*this = copy;
}
~ResourceData()
~FileData()
{
if (owned && memory) free(memory);
}
@ -168,6 +167,7 @@ public:
};
class FileReaderInterface
{
public:
@ -181,12 +181,8 @@ public:
ptrdiff_t GetLength () const { return Length; }
};
struct FResourceLump;
class FileReader
{
friend struct FResourceLump; // needs access to the private constructor.
FileReaderInterface *mReader = nullptr;
FileReader(const FileReader &r) = delete;
@ -210,13 +206,13 @@ public:
FileReader() {}
FileReader(FileReader &&r)
FileReader(FileReader &&r) noexcept
{
mReader = r.mReader;
r.mReader = nullptr;
}
FileReader& operator =(FileReader &&r)
FileReader& operator =(FileReader &&r) noexcept
{
Close();
mReader = r.mReader;
@ -252,11 +248,7 @@ public:
bool OpenFile(const char *filename, Size start = 0, Size length = -1, bool buffered = false);
bool OpenFilePart(FileReader &parent, Size start, Size length);
bool OpenMemory(const void *mem, Size length); // read directly from the buffer
bool OpenMemoryArray(const void *mem, Size length); // read from a copy of the buffer.
bool OpenMemoryArray(std::vector<uint8_t>& data); // take the given array
bool OpenMemoryArray(ResourceData& data); // take the given array
bool OpenMemoryArray(std::function<bool(std::vector<uint8_t>&)> getter); // read contents to a buffer and return a reader to it
bool OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions = false); // creates a decompressor stream. 'seekable' uses a buffered version so that the Seek and Tell methods can be used.
bool OpenMemoryArray(FileData& data); // take the given array
Size Tell() const
{
@ -273,36 +265,14 @@ public:
return mReader->Read(buffer, len);
}
ResourceData Read(size_t len)
{
ResourceData buffer;
if (len > 0)
{
Size length = mReader->Read(buffer.allocate(len), len);
if ((size_t)length < len) buffer.allocate(length);
}
return buffer;
}
FileData Read(size_t len);
FileData ReadPadded(size_t padding);
ResourceData Read()
FileData Read()
{
return Read(GetLength());
}
ResourceData ReadPadded(size_t padding)
{
auto len = GetLength();
ResourceData buffer;
if (len > 0)
{
auto p = (char*)buffer.allocate(len + padding);
Size length = mReader->Read(p, len);
if (length < len) buffer.clear();
else memset(p + len, 0, padding);
}
return buffer;
}
char *Gets(char *strbuf, Size len)
{

53
source/common/filesystem/include/fs_filesystem.h
View file

@ -22,42 +22,6 @@ union LumpShortName
};
// A lump in memory.
class FileData
{
public:
FileData() { lump = nullptr; }
const void *GetMem () { return lump->Cache; }
size_t GetSize () { return lump->LumpSize; }
const char* GetString () const { return (const char*)lump->Cache; }
const uint8_t* GetBytes() const { return (const uint8_t*)lump->Cache; }
FileData& operator = (const FileData& copy) = delete;
FileData(const FileData& copy)
{
lump = copy.lump;
lump->Lock();
}
~FileData()
{
if (lump) lump->Unlock();
}
private:
FileData(FResourceLump* nlump)
{
lump = nlump;
if (lump) lump->Lock();
}
FResourceLump* lump;
friend class FileSystem;
};
struct FolderEntry
{
const char *name;
@ -129,9 +93,19 @@ public:
FileData ReadFile (const char *name) { return ReadFile (GetNumForName (name)); }
FileData ReadFileFullName(const char* name) { return ReadFile(GetNumForFullName(name)); }
FileReader OpenFileReader(int lump); // opens a reader that redirects to the containing file's one.
FileReader ReopenFileReader(int lump, bool alwayscache = false); // opens an independent reader.
FileReader OpenFileReader(int lump, int readertype, int readerflags); // opens a reader that redirects to the containing file's one.
FileReader OpenFileReader(const char* name);
FileReader ReopenFileReader(const char* name, bool alwayscache = false);
FileReader OpenFileReader(int lump)
{
return OpenFileReader(lump, READER_SHARED, READERFLAG_SEEKABLE);
}
FileReader ReopenFileReader(int lump, bool alwayscache = false)
{
return OpenFileReader(lump, alwayscache ? READER_CACHED : READER_NEW, READERFLAG_SEEKABLE);
}
int FindLump (const char *name, int *lastlump, bool anyns=false); // [RH] Find lumps with duplication
int FindLumpMulti (const char **names, int *lastlump, bool anyns = false, int *nameindex = NULL); // same with multiple possible names
@ -145,8 +119,7 @@ public:
static uint32_t LumpNameHash (const char *name); // [RH] Create hash key from an 8-char name
int FileLength (int lump) const;
int GetFileOffset (int lump); // [RH] Returns offset of lump in the wadfile
ptrdiff_t FileLength (int lump) const;
int GetFileFlags (int lump); // Return the flags for this lump
const char* GetFileShortName(int lump) const;
const char *GetFileFullName (int lump, bool returnshort = true) const; // [RH] Returns the lump's full name

1
source/common/filesystem/include/fs_findfile.h
View file

@ -23,7 +23,6 @@ using FileList = std::vector<FileListEntry>;
struct FCompressedBuffer;
bool ScanDirectory(std::vector<FileListEntry>& list, const char* dirpath, const char* match, bool nosubdir = false, bool readhidden = false);
bool WriteZip(const char* filename, const FCompressedBuffer* content, size_t contentcount);
bool FS_DirEntryExists(const char* pathname, bool* isdir);
inline void FixPathSeparator(char* path)

182
source/common/filesystem/include/resourcefile.h
View file

@ -7,6 +7,7 @@
#include <vector>
#include <string>
#include "fs_files.h"
#include "fs_decompress.h"
namespace FileSys {
@ -18,7 +19,6 @@ void strReplace(std::string& str, const char* from, const char* to);
struct LumpFilterInfo
{
std::vector<std::string> gameTypeFilter; // this can contain multiple entries
std::string dotFilter;
// The following are for checking if the root directory of a zip can be removed.
std::vector<std::string> reservedFolders;
@ -78,125 +78,69 @@ typedef enum {
enum ELumpFlags
{
LUMPF_MAYBEFLAT = 1, // might be a flat outside F_START/END
LUMPF_FULLPATH = 2, // contains a full path. This will trigger extended namespace checks when looking up short names.
LUMPF_EMBEDDED = 4, // marks an embedded resource file for later processing.
LUMPF_SHORTNAME = 8, // the stored name is a short extension-less name
LUMPF_COMPRESSED = 16, // compressed or encrypted, i.e. cannot be read with the container file's reader.
RESFF_MAYBEFLAT = 1, // might be a flat inside a WAD outside F_START/END
RESFF_FULLPATH = 2, // contains a full path. This will trigger extended namespace checks when looking up short names.
RESFF_EMBEDDED = 4, // marks an embedded resource file for later processing.
RESFF_SHORTNAME = 8, // the stored name is a short extension-less name
RESFF_COMPRESSED = 16, // compressed or encrypted, i.e. cannot be read with the container file's reader.
RESFF_NEEDFILESTART = 32, // The real position is not known yet and needs to be calculated on access
};
// This holds a compresed Zip entry with all needed info to decompress it.
struct FCompressedBuffer
enum EReaderType
{
size_t mSize;
size_t mCompressedSize;
int mMethod;
int mZipFlags;
unsigned mCRC32;
char *mBuffer;
const char* filename;
bool Decompress(char *destbuffer);
void Clean()
{
mSize = mCompressedSize = 0;
if (mBuffer != nullptr)
{
delete[] mBuffer;
mBuffer = nullptr;
}
}
READER_SHARED = 0, // returns a view into the parent's reader.
READER_NEW = 1, // opens a new file handle
READER_CACHED = 2, // returns a MemoryArrayReader
READERFLAG_SEEKABLE = 1 // ensure the reader is seekable.
};
struct FResourceLump
struct FResourceEntry
{
protected:
friend class FResourceFile;
friend class FWadFile; // this still needs direct access.
friend class FileData;
friend class FileSystem;
friend class FLumpFile;
friend class FLumpReader;
friend class FGrpFile;
friend class F7ZFile;
friend class FSSIFile;
friend class FWHResFile;
friend class FZipFile;
friend class FPakFile;
friend class FRFFFile;
friend class FDirectory;
friend int lumpcmp(const void* a, const void* b);
int LumpSize;
int RefCount;
//protected:
const char* FullName;
//public:
uint8_t Flags;
char * Cache;
FResourceFile * Owner;
public:
FResourceLump()
{
Cache = NULL;
Owner = NULL;
Flags = 0;
RefCount = 0;
FullName = "";
LumpSize = 0;
}
virtual ~FResourceLump();
protected:
virtual FileReader *GetReader();
virtual FileReader NewReader();
virtual int GetFileOffset() { return -1; }
virtual int GetIndexNum() const { return -1; }
virtual int GetNamespace() const { return 0; }
void LumpNameSetup(const char* iname, StringPool* allocator);
void CheckEmbedded(LumpFilterInfo* lfi);
virtual FCompressedBuffer GetRawData();
void *Lock(); // validates the cache and increases the refcount.
int Unlock(); // decreases the refcount and frees the buffer
unsigned Size() const{ return LumpSize; }
int LockCount() const { return RefCount; }
const char* getName() { return FullName; }
void clearName() { FullName = ""; }
protected:
virtual int FillCache() { return -1; }
size_t Length;
size_t CompressedSize;
const char* FileName;
size_t Position;
int ResourceID;
uint32_t CRC32;
uint16_t Flags;
uint16_t Method;
int16_t Namespace;
};
void SetMainThread();
class FResourceFile
{
public:
FResourceFile(const char* filename, StringPool* sp);
FResourceFile(const char* filename, FileReader& r, StringPool* sp);
const char* NormalizeFileName(const char* fn, int fallbackcp = 0);
FResourceEntry* AllocateEntries(int count);
void GenerateHash();
void PostProcessArchive(LumpFilterInfo* filter);
protected:
FileReader Reader;
const char* FileName;
FResourceEntry* Entries = nullptr;
uint32_t NumLumps;
char Hash[48];
StringPool* stringpool;
FResourceFile(const char *filename, StringPool* sp);
FResourceFile(const char *filename, FileReader &r, StringPool* sp);
// for archives that can contain directories
void GenerateHash();
void PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterInfo *filter);
virtual void SetEntryAddress(uint32_t entry)
{
Entries[entry].Flags &= ~RESFF_NEEDFILESTART;
}
bool IsFileInFolder(const char* const resPath);
void CheckEmbedded(uint32_t entry, LumpFilterInfo* lfi);
private:
uint32_t FirstLump;
int FilterLumps(const std::string& filtername, void *lumps, size_t lumpsize, uint32_t max);
int FilterLumpsByGameType(LumpFilterInfo *filter, void *lumps, size_t lumpsize, uint32_t max);
bool FindPrefixRange(const char* filter, void *lumps, size_t lumpsize, uint32_t max, uint32_t &start, uint32_t &end);
void JunkLeftoverFilters(void *lumps, size_t lumpsize, uint32_t max);
int FilterLumps(const std::string& filtername, uint32_t max);
bool FindPrefixRange(const char* filter, uint32_t max, uint32_t &start, uint32_t &end);
void JunkLeftoverFilters(uint32_t max);
void FindCommonFolder(LumpFilterInfo* filter);
static FResourceFile *DoOpenResourceFile(const char *filename, FileReader &file, bool containeronly, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
public:
@ -211,47 +155,45 @@ public:
void SetFirstLump(uint32_t f) { FirstLump = f; }
const char* GetHash() const { return Hash; }
virtual FResourceLump *GetLump(int no) = 0;
int EntryCount() const { return NumLumps; }
int FindEntry(const char* name);
size_t Length(int entry)
size_t Length(uint32_t entry)
{
auto l = GetLump(entry);
return l ? l->LumpSize : -1;
return (entry < NumLumps) ? Entries[entry].Length : 0;
}
size_t Offset(uint32_t entry)
{
return (entry < NumLumps) ? Entries[entry].Position : 0;
}
FileReader GetEntryReader(int entry, bool newreader = true)
// default is the safest reader type.
virtual FileReader GetEntryReader(uint32_t entry, int readertype = READER_NEW, int flags = READERFLAG_SEEKABLE);
int GetEntryFlags(uint32_t entry)
{
auto l = GetLump(entry);
return l ? l->NewReader() : FileReader();
return (entry < NumLumps) ? Entries[entry].Flags : 0;
}
int GetEntryFlags(int entry)
int GetEntryNamespace(uint32_t entry)
{
auto l = GetLump(entry);
return l ? l->Flags : 0;
return (entry < NumLumps) ? Entries[entry].Namespace : ns_hidden;
}
ResourceData Read(int entry)
int GetEntryResourceID(uint32_t entry)
{
auto fr = GetEntryReader(entry, false);
return fr.Read();
return (entry < NumLumps) ? Entries[entry].ResourceID : -1;
}
const char* getName(int entry)
const char* getName(uint32_t entry)
{
auto l = GetLump(entry);
return l ? l->FullName : nullptr;
}
FCompressedBuffer GetRawData(int entry)
{
auto l = GetLump(entry);
if (!l) return {};
return l->GetRawData();
return (entry < NumLumps) ? Entries[entry].FileName : nullptr;
}
virtual FileData Read(int entry);
virtual FCompressedBuffer GetRawData(uint32_t entry);
FileReader Destroy()
{
auto fr = std::move(Reader);

4
source/common/filesystem/source/w_zip.h → source/common/filesystem/include/w_zip.h
View file

@ -7,7 +7,8 @@
#define FORCE_PACKED
#endif
namespace FileSys {
#include <stdint.h>
#pragma pack(1)
// FZipCentralInfo
@ -107,5 +108,4 @@ struct FZipLocalFileHeader
// File header flags.
#define ZF_ENCRYPTED 0x1
}
#endif

17
source/common/filesystem/source/ancientzip.cpp
View file

@ -44,6 +44,7 @@
#include <stdlib.h>
#include <assert.h>
#include "ancientzip.h"
namespace FileSys {
@ -125,7 +126,7 @@ static const unsigned char BitReverse4[] = {
#define FIRST_BIT_LEN 8
#define REST_BIT_LEN 4
void FZipExploder::InsertCode(TArray<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value)
void FZipExploder::InsertCode(std::vector<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value)
{
assert(len > 0);
unsigned int node = pos + (code & ((1 << bits) - 1));
@ -161,12 +162,12 @@ void FZipExploder::InsertCode(TArray<HuffNode> &decoder, unsigned int pos, int b
}
}
unsigned int FZipExploder::InitTable(TArray<HuffNode> &decoder, int numspots)
unsigned int FZipExploder::InitTable(std::vector<HuffNode> &decoder, int numspots)
{
unsigned int start = decoder.Size();
decoder.Reserve(numspots);
size_t start = decoder.size();
decoder.resize(decoder.size() + numspots);
memset(&decoder[start], 0, sizeof(HuffNode)*numspots);
return start;
return (unsigned)start;
}
int FZipExploder::buildercmp(const void *a, const void *b)
@ -180,7 +181,7 @@ int FZipExploder::buildercmp(const void *a, const void *b)
return d;
}
int FZipExploder::BuildDecoder(TArray<HuffNode> &decoder, TableBuilder *values, int numvals)
int FZipExploder::BuildDecoder(std::vector<HuffNode> &decoder, TableBuilder *values, int numvals)
{
int i;
@ -218,7 +219,7 @@ int FZipExploder::BuildDecoder(TArray<HuffNode> &decoder, TableBuilder *values,
}
int FZipExploder::DecodeSFValue(const TArray<HuffNode> &decoder)
int FZipExploder::DecodeSFValue(const std::vector<HuffNode> &decoder)
{
unsigned int bits = FIRST_BIT_LEN, table = 0, code;
const HuffNode *pos;
@ -236,7 +237,7 @@ int FZipExploder::DecodeSFValue(const TArray<HuffNode> &decoder)
}
int FZipExploder::DecodeSF(TArray<HuffNode> &decoder, int numvals)
int FZipExploder::DecodeSF(std::vector<HuffNode> &decoder, int numvals)
{
TableBuilder builder[256];
unsigned char a, c;

16
source/common/filesystem/source/ancientzip.h
View file

@ -27,18 +27,18 @@ class FZipExploder
unsigned short Code;
};
TArray<HuffNode> LiteralDecoder;
TArray<HuffNode> DistanceDecoder;
TArray<HuffNode> LengthDecoder;
std::vector<HuffNode> LiteralDecoder;
std::vector<HuffNode> DistanceDecoder;
std::vector<HuffNode> LengthDecoder;
unsigned char ReadBuf[256];
unsigned int bs, be;
static int buildercmp(const void *a, const void *b);
void InsertCode(TArray<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value);
unsigned int InitTable(TArray<HuffNode> &decoder, int numspots);
int BuildDecoder(TArray<HuffNode> &decoder, TableBuilder *values, int numvals);
int DecodeSFValue(const TArray<HuffNode> &currentTree);
int DecodeSF(TArray<HuffNode> &decoder, int numvals);
void InsertCode(std::vector<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value);
unsigned int InitTable(std::vector<HuffNode> &decoder, int numspots);
int BuildDecoder(std::vector<HuffNode> &decoder, TableBuilder *values, int numvals);
int DecodeSFValue(const std::vector<HuffNode> &currentTree);
int DecodeSF(std::vector<HuffNode> &decoder, int numvals);
public:
int Explode(unsigned char *out, unsigned int outsize, FileReader &in, unsigned int insize, int flags);
};

150
source/common/filesystem/source/critsec.cpp Normal file
View file

@ -0,0 +1,150 @@
/*
**
**
**---------------------------------------------------------------------------
** Copyright 2005-2016 Randy Heit
** All rights reserved.
**
** Redistribution and use in source and binary forms, with or without
** modification, are permitted provided that the following conditions
** are met:
**
** 1. Redistributions of source code must retain the above copyright
** notice, this list of conditions and the following disclaimer.
** 2. Redistributions in binary form must reproduce the above copyright
** notice, this list of conditions and the following disclaimer in the
** documentation and/or other materials provided with the distribution.
** 3. The name of the author may not be used to endorse or promote products
** derived from this software without specific prior written permission.
**
** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**---------------------------------------------------------------------------
**
*/
namespace FileSys {
#ifdef _WIN32
#ifndef _WINNT_
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#endif
class FInternalCriticalSection
{
public:
void Enter()
{
AcquireSRWLockExclusive(&CritSec);
}
void Leave()
{
ReleaseSRWLockExclusive(&CritSec);
}
private:
SRWLOCK CritSec = SRWLOCK_INIT;
};
FInternalCriticalSection *CreateCriticalSection()
{
return new FInternalCriticalSection();
}
void DeleteCriticalSection(FInternalCriticalSection *c)
{
delete c;
}
void EnterCriticalSection(FInternalCriticalSection *c)
{
c->Enter();
}
void LeaveCriticalSection(FInternalCriticalSection *c)
{
c->Leave();
}
#else
#include "critsec.h"
#include <pthread.h>
class FInternalCriticalSection
{
public:
FInternalCriticalSection();
~FInternalCriticalSection();
void Enter();
void Leave();
private:
pthread_mutex_t m_mutex;
};
// TODO: add error handling
FInternalCriticalSection::FInternalCriticalSection()
{
pthread_mutexattr_t attributes;
pthread_mutexattr_init(&attributes);
pthread_mutexattr_settype(&attributes, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&m_mutex, &attributes);
pthread_mutexattr_destroy(&attributes);
}
FInternalCriticalSection::~FInternalCriticalSection()
{
pthread_mutex_destroy(&m_mutex);
}
void FInternalCriticalSection::Enter()
{
pthread_mutex_lock(&m_mutex);
}
void FInternalCriticalSection::Leave()
{
pthread_mutex_unlock(&m_mutex);
}
FInternalCriticalSection *CreateCriticalSection()
{
return new FInternalCriticalSection();
}
void DeleteCriticalSection(FInternalCriticalSection *c)
{
delete c;
}
void EnterCriticalSection(FInternalCriticalSection *c)
{
c->Enter();
}
void LeaveCriticalSection(FInternalCriticalSection *c)
{
c->Leave();
}
#endif
}

39
source/common/filesystem/source/critsec.h Normal file
View file

@ -0,0 +1,39 @@
#pragma once
namespace FileSys {
// System independent critical sections without polluting the namespace with the operating system headers.
class FInternalCriticalSection;
FInternalCriticalSection *CreateCriticalSection();
void DeleteCriticalSection(FInternalCriticalSection *c);
void EnterCriticalSection(FInternalCriticalSection *c);
void LeaveCriticalSection(FInternalCriticalSection *c);
// This is just a convenience wrapper around the function interface adjusted to use std::lock_guard
class FCriticalSection
{
public:
FCriticalSection()
{
c = CreateCriticalSection();
}
~FCriticalSection()
{
DeleteCriticalSection(c);
}
void lock()
{
EnterCriticalSection(c);
}
void unlock()
{
LeaveCriticalSection(c);
}
private:
FInternalCriticalSection *c;
};
}

102
source/common/filesystem/source/file_7z.cpp
View file

@ -38,6 +38,9 @@
#include "7zCrc.h"
#include "resourcefile.h"
#include "fs_findfile.h"
#include "unicode.h"
#include "critsec.h"
#include <mutex>
namespace FileSys {
@ -158,20 +161,6 @@ struct C7zArchive
return res;
}
};
//==========================================================================
//
// Zip Lump
//
//==========================================================================
struct F7ZLump : public FResourceLump
{
int Position;
virtual int FillCache() override;
};
//==========================================================================
//
@ -183,14 +172,15 @@ class F7ZFile : public FResourceFile
{
friend struct F7ZLump;
F7ZLump *Lumps;
C7zArchive *Archive;
FCriticalSection critsec;
public:
F7ZFile(const char * filename, FileReader &filer, StringPool* sp);
bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf);
virtual ~F7ZFile();
virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; }
FileData Read(int entry) override;
FileReader GetEntryReader(uint32_t entry, int, int) override;
};
@ -204,8 +194,7 @@ public:
F7ZFile::F7ZFile(const char * filename, FileReader &filer, StringPool* sp)
: FResourceFile(filename, filer, sp)
{
Lumps = NULL;
Archive = NULL;
Archive = nullptr;
}
@ -247,12 +236,11 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
CSzArEx* const archPtr = &Archive->DB;
AllocateEntries(archPtr->NumFiles);
NumLumps = archPtr->NumFiles;
Lumps = new F7ZLump[NumLumps];
F7ZLump *lump_p = Lumps;
std::u16string nameUTF16;
std::string nameASCII;
std::vector<char> nameASCII;
for (uint32_t i = 0; i < NumLumps; ++i)
{
@ -273,21 +261,17 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
nameUTF16.resize((unsigned)nameLength);
nameASCII.resize((unsigned)nameLength);
// note that the file system is not equipped to handle non-ASCII, so don't bother with proper Unicode conversion here.
SzArEx_GetFileNameUtf16(archPtr, i, (UInt16*)nameUTF16.data());
for (size_t c = 0; c < nameLength; ++c)
{
nameASCII[c] = tolower(static_cast<char>(nameUTF16[c]));
}
FixPathSeparator(&nameASCII.front());
lump_p->LumpNameSetup(nameASCII.c_str(), stringpool);
lump_p->LumpSize = static_cast<int>(SzArEx_GetFileSize(archPtr, i));
lump_p->Owner = this;
lump_p->Flags = LUMPF_FULLPATH|LUMPF_COMPRESSED;
lump_p->Position = i;
lump_p->CheckEmbedded(filter);
lump_p++;
SzArEx_GetFileNameUtf16(archPtr, i, (UInt16*)nameUTF16.data());
utf16_to_utf8((uint16_t*)nameUTF16.data(), nameASCII);
Entries[i].FileName = NormalizeFileName(nameASCII.data());
Entries[i].Length = SzArEx_GetFileSize(archPtr, i);
Entries[i].Flags = RESFF_FULLPATH|RESFF_COMPRESSED;
Entries[i].ResourceID = -1;
Entries[i].Namespace = ns_global;
Entries[i].Method = METHOD_INVALID;
Entries[i].Position = i;
}
// Resize the lump record array to its actual size
NumLumps -= skipped;
@ -296,10 +280,9 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
{
// Quick check for unsupported compression method
TArray<char> temp;
temp.Resize(Lumps[0].LumpSize);
FileData temp(nullptr, Entries[0].Length);
if (SZ_OK != Archive->Extract(Lumps[0].Position, &temp[0]))
if (SZ_OK != Archive->Extract(Entries[0].Position, (char*)temp.writable()))
{
Printf(FSMessageLevel::Error, "%s: unsupported 7z/LZMA file!\n", FileName);
return false;
@ -307,7 +290,7 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
}
GenerateHash();
PostProcessArchive(&Lumps[0], sizeof(F7ZLump), filter);
PostProcessArchive(filter);
return true;
}
@ -319,11 +302,7 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf)
F7ZFile::~F7ZFile()
{
if (Lumps != NULL)
{
delete[] Lumps;
}
if (Archive != NULL)
if (Archive != nullptr)
{
delete Archive;
}
@ -331,22 +310,41 @@ F7ZFile::~F7ZFile()
//==========================================================================
//
// Fills the lump cache and performs decompression
// Reads data for one entry into a buffer
//
//==========================================================================
int F7ZLump::FillCache()
FileData F7ZFile::Read(int entry)
{
Cache = new char[LumpSize];
SRes code = static_cast<F7ZFile*>(Owner)->Archive->Extract(Position, Cache);
if (code != SZ_OK)
FileData buffer;
if ((entry >= 0 || entry < NumLumps) && Entries[entry].Length > 0)
{
throw FileSystemException("Error %d reading from 7z archive", code);
auto p = buffer.allocate(Entries[entry].Length);
// There is no realistic way to keep multiple references to a 7z file open without massive overhead so to make this thread-safe a mutex is the only option.
std::lock_guard<FCriticalSection> lock(critsec);
SRes code = Archive->Extract(Entries[entry].Position, (char*)p);
if (code != SZ_OK) buffer.clear();
}
RefCount = 1;
return 1;
return buffer;
}
//==========================================================================
//
// This can only return a FileReader to a memory buffer.
//
//==========================================================================
FileReader F7ZFile::GetEntryReader(uint32_t entry, int, int)
{
FileReader fr;
if (entry < 0 || entry >= NumLumps) return fr;
auto buffer = Read(entry);
if (buffer.size() > 0)
fr.OpenMemoryArray(buffer);
return fr;
}
//==========================================================================
//
// File open

101
source/common/filesystem/source/file_directory.cpp
View file

@ -48,21 +48,6 @@ std::string FS_FullPath(const char* directory);
std::wstring toWide(const char* str);
#endif
//==========================================================================
//
// Zip Lump
//
//==========================================================================
struct FDirectoryLump : public FResourceLump
{
FileReader NewReader() override;
int FillCache() override;
const char* mFullPath;
};
//==========================================================================
//
// Zip file
@ -71,16 +56,15 @@ struct FDirectoryLump : public FResourceLump
class FDirectory : public FResourceFile
{
TArray<FDirectoryLump> Lumps;
const bool nosubdir;
const char* mBasePath;
int AddDirectory(const char* dirpath, LumpFilterInfo* filter, FileSystemMessageFunc Printf);
void AddEntry(const char *fullpath, const char* relpath, int size);
public:
FDirectory(const char * dirname, StringPool* sp, bool nosubdirflag = false);
bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf);
virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; }
FileReader GetEntryReader(uint32_t entry, int, int) override;
};
@ -116,8 +100,17 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy
}
else
{
mBasePath = nullptr;
AllocateEntries(list.size());
for(auto& entry : list)
{
if (mBasePath == nullptr)
{
// extract the base path from the first entry to cover changes made in ScanDirectory.
auto full = entry.FilePath.find(entry.FilePathRel);
std::string path(entry.FilePath, 0, full);
mBasePath = stringpool->Strdup(path.c_str());
}
if (!entry.isDirectory)
{
auto fi = entry.FileName;
@ -128,6 +121,7 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy
continue;
}
if (filter->filenamecheck == nullptr || filter->filenamecheck(fi.c_str(), entry.FilePath.c_str()))
{
if (entry.Length > 0x7fffffff)
@ -135,7 +129,13 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy
Printf(FSMessageLevel::Warning, "%s is larger than 2GB and will be ignored\n", entry.FilePath.c_str());
continue;
}
AddEntry(entry.FilePath.c_str(), entry.FilePathRel.c_str(), (int)entry.Length);
// for internal access we use the normalized form of the relative path.
Entries[count].FileName = NormalizeFileName(entry.FilePathRel.c_str());
Entries[count].Length = entry.Length;
Entries[count].Flags = RESFF_FULLPATH;
Entries[count].ResourceID = -1;
Entries[count].Method = METHOD_STORED;
Entries[count].Namespace = ns_global;
count++;
}
}
@ -153,7 +153,6 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy
bool FDirectory::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
{
NumLumps = AddDirectory(FileName, filter, Printf);
PostProcessArchive(&Lumps[0], sizeof(FDirectoryLump), filter);
return true;
}
@ -163,62 +162,22 @@ bool FDirectory::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
//
//==========================================================================
void FDirectory::AddEntry(const char *fullpath, const char* relpath, int size)
{
FDirectoryLump *lump_p = &Lumps[Lumps.Reserve(1)];
// Store the full path here so that we can access the file later, even if it is from a filter directory.
lump_p->mFullPath = stringpool->Strdup(fullpath);
// [mxd] Convert name to lowercase
std::string name = relpath;
for (auto& c : name) c = tolower(c);
// The lump's name is only the part relative to the main directory
lump_p->LumpNameSetup(name.c_str(), stringpool);
lump_p->LumpSize = size;
lump_p->Owner = this;
lump_p->Flags = 0;
lump_p->CheckEmbedded(nullptr);
}
//==========================================================================
//
//
//
//==========================================================================
FileReader FDirectoryLump::NewReader()
FileReader FDirectory::GetEntryReader(uint32_t entry, int readertype, int)
{
FileReader fr;
fr.OpenFile(mFullPath);
if (entry < NumLumps)
{
std::string fn = mBasePath; fn += Entries[entry].FileName;
fr.OpenFile(fn.c_str());
if (readertype == READER_CACHED)
{
auto data = fr.Read();
fr.OpenMemoryArray(data);
}
}
return fr;
}
//==========================================================================
//
//
//
//==========================================================================
int FDirectoryLump::FillCache()
{
FileReader fr;
Cache = new char[LumpSize];
if (!fr.OpenFile(mFullPath))
{
throw FileSystemException("unable to open file");
}
auto read = fr.Read(Cache, LumpSize);
if (read != LumpSize)
{
throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize);
}
RefCount = 1;
return 1;
}
//==========================================================================
//
// File open

59
source/common/filesystem/source/file_grp.cpp
View file

@ -33,7 +33,7 @@
**
*/
#include "resourcefile_internal.h"
#include "resourcefile.h"
#include "fs_swap.h"
namespace FileSys {
@ -65,62 +65,39 @@ struct GrpLump
};
//==========================================================================
//
// Build GRP file
//
//==========================================================================
class FGrpFile : public FUncompressedFile
{
public:
FGrpFile(const char * filename, FileReader &file, StringPool* sp);
bool Open(LumpFilterInfo* filter);
};
//==========================================================================
//
// Initializes a Build GRP file
//
//==========================================================================
FGrpFile::FGrpFile(const char *filename, FileReader &file, StringPool* sp)
: FUncompressedFile(filename, file, sp)
{
}
//==========================================================================
//
// Open it
//
//==========================================================================
bool FGrpFile::Open(LumpFilterInfo* filter)
static bool OpenGrp(FResourceFile* file, LumpFilterInfo* filter)
{
GrpHeader header;
Reader.Read(&header, sizeof(header));
NumLumps = LittleLong(header.NumLumps);
auto Reader = file->GetContainerReader();
Reader->Read(&header, sizeof(header));
uint32_t NumLumps = LittleLong(header.NumLumps);
auto Entries = file->AllocateEntries(NumLumps);
GrpLump *fileinfo = new GrpLump[NumLumps];
Reader.Read (fileinfo, NumLumps * sizeof(GrpLump));
Lumps.Resize(NumLumps);
Reader->Read (fileinfo, NumLumps * sizeof(GrpLump));
int Position = sizeof(GrpHeader) + NumLumps * sizeof(GrpLump);
for(uint32_t i = 0; i < NumLumps; i++)
{
Lumps[i].Owner = this;
Lumps[i].Position = Position;
Lumps[i].LumpSize = LittleLong(fileinfo[i].Size);
Entries[i].Position = Position;
Entries[i].Length = LittleLong(fileinfo[i].Size);
Position += fileinfo[i].Size;
Lumps[i].Flags = 0;
Entries[i].Flags = 0;
Entries[i].Namespace = ns_global;
fileinfo[i].NameWithZero[12] = '\0'; // Be sure filename is null-terminated
Lumps[i].LumpNameSetup(fileinfo[i].NameWithZero, stringpool);
Entries[i].ResourceID = -1;
Entries[i].Method = METHOD_STORED;
Entries[i].FileName = file->NormalizeFileName(fileinfo[i].Name);
}
GenerateHash();
file->GenerateHash();
delete[] fileinfo;
return true;
}
@ -143,12 +120,12 @@ FResourceFile *CheckGRP(const char *filename, FileReader &file, LumpFilterInfo*
file.Seek(0, FileReader::SeekSet);
if (!memcmp(head, "KenSilverman", 12))
{
auto rf = new FGrpFile(filename, file, sp);
if (rf->Open(filter)) return rf;
auto rf = new FResourceFile(filename, file, sp);
if (OpenGrp(rf, filter)) return rf;
file = rf->Destroy();
}
}
return NULL;
return nullptr;
}
}

109
source/common/filesystem/source/file_hog.cpp Normal file
View file

@ -0,0 +1,109 @@
/*
** file_hog.cpp
**
** reads Descent .hog files
**
**---------------------------------------------------------------------------
** Copyright 2023 Christoph Oelckers
** All rights reserved.
**
** Redistribution and use in source and binary forms, with or without
** modification, are permitted provided that the following conditions
** are met:
**
** 1. Redistributions of source code must retain the above copyright
** notice, this list of conditions and the following disclaimer.
** 2. Redistributions in binary form must reproduce the above copyright
** notice, this list of conditions and the following disclaimer in the
** documentation and/or other materials provided with the distribution.
** 3. The name of the author may not be used to endorse or promote products
** derived from this software without specific prior written permission.
**
** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**---------------------------------------------------------------------------
**
**
*/
#include "resourcefile.h"
#include "fs_swap.h"
namespace FileSys {
using namespace byteswap;
static bool OpenHog(FResourceFile* rf, LumpFilterInfo* filter)
{
auto Reader = rf->GetContainerReader();
FileReader::Size length = Reader->GetLength();
std::vector<FResourceEntry> entries;
// Hogs store their data as a list of file records, each containing a name, length and the actual data.
// To read the directory the entire file must be scanned.
while (Reader->Tell() <= length)
{
char name[13];
auto r = Reader->Read(&name, 13);
if (r < 13) break;
name[12] = 0;
uint32_t elength = Reader->ReadUInt32();
FResourceEntry Entry;
Entry.Position = Reader->Tell();
Entry.Length = elength;
Entry.Flags = 0;
Entry.CRC32 = 0;
Entry.Namespace = ns_global;
Entry.ResourceID = -1;
Entry.Method = METHOD_STORED;
Entry.FileName = rf->NormalizeFileName(name);
entries.push_back(Entry);
Reader->Seek(elength, FileReader::SeekCur);
}
auto Entries = rf->AllocateEntries((int)entries.size());
memcpy(Entries, entries.data(), entries.size() * sizeof(Entries[0]));
rf->GenerateHash();
return true;
}
//==========================================================================
//
// File open
//
//==========================================================================
FResourceFile* CheckHog(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp)
{
char head[3];
if (file.GetLength() >= 20)
{
file.Seek(0, FileReader::SeekSet);
file.Read(&head, 3);
if (!memcmp(head, "DHF", 3))
{
auto rf = new FResourceFile(filename, file, sp);
if (OpenHog(rf, filter)) return rf;
file = rf->Destroy();
}
file.Seek(0, FileReader::SeekSet);
}
return nullptr;
}
}

48
source/common/filesystem/source/file_lump.cpp
View file

@ -32,49 +32,25 @@
**
*/
#include "resourcefile_internal.h"
#include "resourcefile.h"
namespace FileSys {
//==========================================================================
//
// Single lump
//
//==========================================================================
class FLumpFile : public FUncompressedFile
{
public:
FLumpFile(const char * filename, FileReader &file, StringPool* sp);
bool Open(LumpFilterInfo* filter);
};
//==========================================================================
//
// FLumpFile::FLumpFile
//
//==========================================================================
FLumpFile::FLumpFile(const char *filename, FileReader &file, StringPool* sp)
: FUncompressedFile(filename, file, sp)
{
}
//==========================================================================
//
// Open it
//
//==========================================================================
bool FLumpFile::Open(LumpFilterInfo*)
static bool OpenLump(FResourceFile* file, LumpFilterInfo*)
{
Lumps.Resize(1);
Lumps[0].LumpNameSetup(ExtractBaseName(FileName, true).c_str(), stringpool);
Lumps[0].Owner = this;
Lumps[0].Position = 0;
Lumps[0].LumpSize = (int)Reader.GetLength();
Lumps[0].Flags = 0;
NumLumps = 1;
auto Entries = file->AllocateEntries(1);
Entries[0].FileName = file->NormalizeFileName(ExtractBaseName(file->GetFileName(), true).c_str());
Entries[0].Namespace = ns_global;
Entries[0].ResourceID = -1;
Entries[0].Position = 0;
Entries[0].Length = file->GetContainerReader()->GetLength();
Entries[0].Method = METHOD_STORED;
Entries[0].Flags = 0;
return true;
}
@ -87,8 +63,8 @@ bool FLumpFile::Open(LumpFilterInfo*)
FResourceFile *CheckLump(const char *filename, FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp)
{
// always succeeds
auto rf = new FLumpFile(filename, file, sp);
if (rf->Open(filter)) return rf;
auto rf = new FResourceFile(filename, file, sp);
if (OpenLump(rf, filter)) return rf;
file = rf->Destroy();
return NULL;
}

98
source/common/filesystem/source/file_mvl.cpp Normal file
View file

@ -0,0 +1,98 @@
/*
** file_mvl.cpp
**
** reads Descent2 .mvl files
**
**---------------------------------------------------------------------------
** Copyright 2023 Christoph Oelckers
** All rights reserved.
**
** Redistribution and use in source and binary forms, with or without
** modification, are permitted provided that the following conditions
** are met:
**
** 1. Redistributions of source code must retain the above copyright
** notice, this list of conditions and the following disclaimer.
** 2. Redistributions in binary form must reproduce the above copyright
** notice, this list of conditions and the following disclaimer in the
** documentation and/or other materials provided with the distribution.
** 3. The name of the author may not be used to endorse or promote products
** derived from this software without specific prior written permission.
**
** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**---------------------------------------------------------------------------
**
**
*/
#include "resourcefile.h"
#include "fs_swap.h"
namespace FileSys {
using namespace byteswap;
static bool OpenMvl(FResourceFile* rf, LumpFilterInfo* filter)
{
auto Reader = rf->GetContainerReader();
auto count = Reader->ReadUInt32();
auto Entries = rf->AllocateEntries(count);
size_t pos = 8 + (17 * count); // files start after the directory
for (uint32_t i = 0; i < count; i++)
{
char name[13];
Reader->Read(&name, 13);
name[12] = 0;
uint32_t elength = Reader->ReadUInt32();
Entries[i].Position = pos;
Entries[i].Length = elength;
Entries[i].ResourceID = -1;
Entries[i].FileName = rf->NormalizeFileName(name);
pos += elength;
}
return true;
}
//==========================================================================
//
// File open
//
//==========================================================================
FResourceFile* CheckMvl(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp)
{
char head[4];
if (file.GetLength() >= 20)
{
file.Seek(0, FileReader::SeekSet);
file.Read(&head, 4);
if (!memcmp(head, "DMVL", 4))
{
auto rf = new FResourceFile(filename, file, sp);
if (OpenMvl(rf, filter)) return rf;
file = rf->Destroy();
}
file.Seek(0, FileReader::SeekSet);
}
return nullptr;
}
}

66
source/common/filesystem/source/file_pak.cpp
View file

@ -32,7 +32,7 @@
**
*/
#include "resourcefile_internal.h"
#include "resourcefile.h"
namespace FileSys {
@ -57,64 +57,38 @@ struct dpackheader_t
} ;
//==========================================================================
//
// Wad file
//
//==========================================================================
class FPakFile : public FUncompressedFile
{
public:
FPakFile(const char * filename, FileReader &file, StringPool* sp);
bool Open(LumpFilterInfo* filter);
};
//==========================================================================
//
// FWadFile::FWadFile
//
// Initializes a WAD file
//
//==========================================================================
FPakFile::FPakFile(const char *filename, FileReader &file, StringPool* sp)
: FUncompressedFile(filename, file, sp)
{
}
//==========================================================================
//
// Open it
//
//==========================================================================
bool FPakFile::Open(LumpFilterInfo* filter)
static bool OpenPak(FResourceFile* file, LumpFilterInfo* filter)
{
dpackheader_t header;
Reader.Read(&header, sizeof(header));
NumLumps = LittleLong(header.dirlen) / sizeof(dpackfile_t);
auto Reader = file->GetContainerReader();
Reader->Read(&header, sizeof(header));
uint32_t NumLumps = header.dirlen / sizeof(dpackfile_t);
auto Entries = file->AllocateEntries(NumLumps);
header.dirofs = LittleLong(header.dirofs);
TArray<dpackfile_t> fileinfo(NumLumps, true);
Reader.Seek (header.dirofs, FileReader::SeekSet);
Reader.Read (fileinfo.Data(), NumLumps * sizeof(dpackfile_t));
Lumps.Resize(NumLumps);
Reader->Seek (header.dirofs, FileReader::SeekSet);
auto fd = Reader->Read (NumLumps * sizeof(dpackfile_t));
auto fileinfo = (const dpackfile_t*)fd.data();
for(uint32_t i = 0; i < NumLumps; i++)
{
Lumps[i].LumpNameSetup(fileinfo[i].name, stringpool);
Lumps[i].Flags = LUMPF_FULLPATH;
Lumps[i].Owner = this;
Lumps[i].Position = LittleLong(fileinfo[i].filepos);
Lumps[i].LumpSize = LittleLong(fileinfo[i].filelen);
Lumps[i].CheckEmbedded(filter);
Entries[i].Position = LittleLong(fileinfo[i].filepos);
Entries[i].Length = LittleLong(fileinfo[i].filelen);
Entries[i].Flags = RESFF_FULLPATH;
Entries[i].Namespace = ns_global;
Entries[i].ResourceID = -1;
Entries[i].Method = METHOD_STORED;
Entries[i].FileName = file->NormalizeFileName(fileinfo[i].name);
}
GenerateHash();
PostProcessArchive(&Lumps[0], sizeof(Lumps[0]), filter);
file->GenerateHash();
file->PostProcessArchive(filter);
return true;
}
@ -136,8 +110,8 @@ FResourceFile *CheckPak(const char *filename, FileReader &file, LumpFilterInfo*
file.Seek(0, FileReader::SeekSet);
if (!memcmp(head, "PACK", 4))
{
auto rf = new FPakFile(filename, file, sp);
if (rf->Open(filter)) return rf;
auto rf = new FResourceFile(filename, file, sp);
if (OpenPak(rf, filter)) return rf;
file = rf->Destroy();
}
}

145
source/common/filesystem/source/file_rff.cpp
View file

@ -33,7 +33,8 @@
**
*/
#include "resourcefile_internal.h"
#include <assert.h>
#include "resourcefile.h"
#include "fs_swap.h"
namespace FileSys {
@ -67,22 +68,6 @@ struct RFFLump
uint32_t IndexNum; // Used by .sfx, possibly others
};
//==========================================================================
//
// Blood RFF lump (uncompressed lump with encryption)
//
//==========================================================================
struct FRFFLump : public FUncompressedLump
{
virtual FileReader *GetReader();
virtual int FillCache() override;
uint32_t IndexNum;
int GetIndexNum() const { return IndexNum; }
};
//==========================================================================
//
// BloodCrypt
@ -100,68 +85,47 @@ void BloodCrypt (void *data, int key, int len)
}
//==========================================================================
//
// Blood RFF file
//
//==========================================================================
class FRFFFile : public FResourceFile
{
FRFFLump *Lumps;
public:
FRFFFile(const char * filename, FileReader &file, StringPool* sp);
virtual ~FRFFFile();
virtual bool Open(LumpFilterInfo* filter);
virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; }
};
//==========================================================================
//
// Initializes a Blood RFF file
//
//==========================================================================
FRFFFile::FRFFFile(const char *filename, FileReader &file, StringPool* sp)
: FResourceFile(filename, file, sp)
{
Lumps = NULL;
}
//==========================================================================
//
// Initializes a Blood RFF file
//
//==========================================================================
bool FRFFFile::Open(LumpFilterInfo*)
static bool OpenRFF(FResourceFile* file, LumpFilterInfo*)
{
RFFLump *lumps;
RFFInfo header;
Reader.Read(&header, sizeof(header));
auto Reader = file->GetContainerReader();
Reader->Read(&header, sizeof(header));
NumLumps = LittleLong(header.NumLumps);
uint32_t NumLumps = LittleLong(header.NumLumps);
auto Entries = file->AllocateEntries(NumLumps);
header.DirOfs = LittleLong(header.DirOfs);
lumps = new RFFLump[header.NumLumps];
Reader.Seek (header.DirOfs, FileReader::SeekSet);
Reader.Read (lumps, header.NumLumps * sizeof(RFFLump));
BloodCrypt (lumps, header.DirOfs, header.NumLumps * sizeof(RFFLump));
Lumps = new FRFFLump[NumLumps];
Reader->Seek (LittleLong(header.DirOfs), FileReader::SeekSet);
Reader->Read (lumps, NumLumps * sizeof(RFFLump));
BloodCrypt (lumps, LittleLong(header.DirOfs), NumLumps * sizeof(RFFLump));
for (uint32_t i = 0; i < NumLumps; ++i)
{
Lumps[i].Position = LittleLong(lumps[i].FilePos);
Lumps[i].LumpSize = LittleLong(lumps[i].Size);
Lumps[i].Owner = this;
Entries[i].Position = LittleLong(lumps[i].FilePos);
Entries[i].Length = LittleLong(lumps[i].Size);
Entries[i].Flags = 0;
Entries[i].Method = METHOD_STORED;
if (lumps[i].Flags & 0x10)
{
Lumps[i].Flags |= LUMPF_COMPRESSED; // flags the lump as not directly usable
Entries[i].Flags = RESFF_COMPRESSED; // for purposes of decoding, compression and encryption are equivalent.
Entries[i].Method = METHOD_RFFCRYPT;
}
Lumps[i].IndexNum = LittleLong(lumps[i].IndexNum);
else
{
Entries[i].Flags = 0;
Entries[i].Method = METHOD_STORED;
}
Entries[i].Namespace = ns_global;
Entries[i].ResourceID = LittleLong(lumps[i].IndexNum);
// Rearrange the name and extension to construct the fullname.
char name[13];
strncpy(name, lumps[i].Name, 8);
@ -173,66 +137,13 @@ bool FRFFFile::Open(LumpFilterInfo*)
name[len+2] = lumps[i].Extension[1];
name[len+3] = lumps[i].Extension[2];
name[len+4] = 0;
Lumps[i].LumpNameSetup(name, stringpool);
Entries[i].FileName = file->NormalizeFileName(name);
}
delete[] lumps;
GenerateHash();
file->GenerateHash();
return true;
}
FRFFFile::~FRFFFile()
{
if (Lumps != NULL)
{
delete[] Lumps;
}
}
//==========================================================================
//
// Get reader (only returns non-NULL if not encrypted)
//
//==========================================================================
FileReader *FRFFLump::GetReader()
{
// Don't return the reader if this lump is encrypted
// In that case always force caching of the lump
if (!(Flags & LUMPF_COMPRESSED))
{
return FUncompressedLump::GetReader();
}
else
{
return NULL;
}
}
//==========================================================================
//
// Fills the lump cache and performs decryption
//
//==========================================================================
int FRFFLump::FillCache()
{
int res = FUncompressedLump::FillCache();
if (Flags & LUMPF_COMPRESSED)
{
int cryptlen = std::min<int> (LumpSize, 256);
uint8_t *data = (uint8_t *)Cache;
for (int i = 0; i < cryptlen; ++i)
{
data[i] ^= i >> 1;
}
}
return res;
}
//==========================================================================
//
// File open
@ -250,8 +161,8 @@ FResourceFile *CheckRFF(const char *filename, FileReader &file, LumpFilterInfo*
file.Seek(0, FileReader::SeekSet);
if (!memcmp(head, "RFF\x1a", 4))
{
auto rf = new FRFFFile(filename, file, sp);
if (rf->Open(filter)) return rf;
auto rf = new FResourceFile(filename, file, sp);
if (OpenRFF(rf, filter)) return rf;
file = rf->Destroy();
}
}

76
source/common/filesystem/source/file_ssi.cpp
View file

@ -33,33 +33,9 @@
**
*/
#include "resourcefile_internal.h"
#include "resourcefile.h"
namespace FileSys {
//==========================================================================
//
// Build GRP file
//
//==========================================================================
class FSSIFile : public FUncompressedFile
{
public:
FSSIFile(const char * filename, FileReader &file, StringPool* sp);
bool Open(int version, int EntryCount, LumpFilterInfo* filter);
};
//==========================================================================
//
// Initializes a Build GRP file
//
//==========================================================================
FSSIFile::FSSIFile(const char *filename, FileReader &file, StringPool* sp)
: FUncompressedFile(filename, file, sp)
{
}
//==========================================================================
//
@ -68,42 +44,50 @@ FSSIFile::FSSIFile(const char *filename, FileReader &file, StringPool* sp)
//
//==========================================================================
bool FSSIFile::Open(int version, int EntryCount, LumpFilterInfo*)
static bool OpenSSI(FResourceFile* file, int version, int EntryCount, LumpFilterInfo*)
{
NumLumps = EntryCount*2;
Lumps.Resize(EntryCount*2);
uint32_t NumLumps = EntryCount * 2;
auto Entries = file->AllocateEntries(NumLumps);
auto Reader = file->GetContainerReader();
int32_t j = (version == 2 ? 267 : 254) + (EntryCount * 121);
for (uint32_t i = 0; i < NumLumps; i+=2)
{
char fn[13];
int strlength = Reader.ReadUInt8();
int strlength = Reader->ReadUInt8();
if (strlength > 12) strlength = 12;
Reader.Read(fn, 12);
Reader->Read(fn, 12);
fn[strlength] = 0;
int flength = Reader.ReadInt32();
int flength = Reader->ReadInt32();
Lumps[i].LumpNameSetup(fn, stringpool);
Lumps[i].Position = j;
Lumps[i].LumpSize = flength;
Lumps[i].Owner = this;
if (strstr(fn, ".GRP")) Lumps[i].Flags |= LUMPF_EMBEDDED;
Entries[i].Position = j;
Entries[i].Length = flength;
Entries[i].Flags = 0;
Entries[i].Namespace = ns_global;
Entries[i].Method = METHOD_STORED;
Entries[i].ResourceID = -1;
Entries[i].FileName = file->NormalizeFileName(fn);
if (strstr(fn, ".GRP")) Entries[i].Flags |= RESFF_EMBEDDED;
// SSI files can swap the order of the extension's characters - but there's no reliable detection for this and it can be mixed inside the same container,
// so we have no choice but to create another file record for the altered name.
std::swap(fn[strlength - 1], fn[strlength - 3]);
Lumps[i+1].LumpNameSetup(fn, stringpool);
Lumps[i+1].Position = j;
Lumps[i+1].LumpSize = flength;
Lumps[i+1].Owner = this;
if (strstr(fn, ".GRP")) Lumps[i+1].Flags |= LUMPF_EMBEDDED;
Entries[i + 1].Position = j;
Entries[i + 1].Length = flength;
Entries[i + 1].Flags = 0;
Entries[i + 1].Namespace = ns_global;
Entries[i + 1].ResourceID = -1;
Entries[i + 1].FileName = file->NormalizeFileName(fn);
Entries[i + 1].Method = METHOD_STORED;
if (strstr(fn, ".GRP")) Entries[i + 1].Flags |= RESFF_EMBEDDED;
j += flength;
Reader.Seek(104, FileReader::SeekCur);
Reader->Seek(104, FileReader::SeekCur);
file->GenerateHash();
}
return true;
}
@ -145,8 +129,8 @@ FResourceFile* CheckSSI(const char* filename, FileReader& file, LumpFilterInfo*
{
if (!skipstring(70)) return nullptr;
}
auto ssi = new FSSIFile(filename, file, sp);
if (ssi->Open(version, numfiles, filter)) return ssi;
auto ssi = new FResourceFile(filename, file, sp);
if (OpenSSI(ssi, version, numfiles, filter)) return ssi;
file = ssi->Destroy();
}
}

209
source/common/filesystem/source/file_wad.cpp
View file

@ -37,6 +37,8 @@
#include "resourcefile.h"
#include "fs_filesystem.h"
#include "fs_swap.h"
#include "fs_stringpool.h"
#include "resourcefile.h"
namespace FileSys {
using namespace byteswap;
@ -56,71 +58,6 @@ struct wadlump_t
char Name[8];
};
//==========================================================================
//
// Wad Lump (with console doom LZSS support)
//
//==========================================================================
class FWadFileLump : public FResourceLump
{
public:
bool Compressed;
int Position;
int Namespace;
int GetNamespace() const override { return Namespace; }
int GetFileOffset() override { return Position; }
FileReader *GetReader() override
{
if(!Compressed)
{
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
return Owner->GetContainerReader();
}
return NULL;
}
int FillCache() override
{
if(!Compressed)
{
const char * buffer = Owner->GetContainerReader()->GetBuffer();
if (buffer != NULL)
{
// This is an in-memory file so the cache can point directly to the file's data.
Cache = const_cast<char*>(buffer) + Position;
RefCount = -1;
return -1;
}
}
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
Cache = new char[LumpSize];
if(Compressed)
{
FileReader lzss;
if (lzss.OpenDecompressor(*Owner->GetContainerReader(), LumpSize, METHOD_LZSS, false, true))
{
lzss.Read(Cache, LumpSize);
}
}
else
{
auto read = Owner->GetContainerReader()->Read(Cache, LumpSize);
if (read != LumpSize)
{
throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize);
}
}
RefCount = 1;
return 1;
}
};
//==========================================================================
//
// Wad file
@ -129,15 +66,12 @@ public:
class FWadFile : public FResourceFile
{
TArray<FWadFileLump> Lumps;
bool IsMarker(int lump, const char *marker);
void SetNamespace(const char *startmarker, const char *endmarker, namespace_t space, FileSystemMessageFunc Printf, bool flathack=false);
void SkinHack (FileSystemMessageFunc Printf);
public:
FWadFile(const char * filename, FileReader &file, StringPool* sp);
FResourceLump *GetLump(int lump) { return &Lumps[lump]; }
bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf);
};
@ -188,56 +122,53 @@ bool FWadFile::Open(LumpFilterInfo*, FileSystemMessageFunc Printf)
}
}
TArray<wadlump_t> fileinfo(NumLumps, true);
Reader.Seek (InfoTableOfs, FileReader::SeekSet);
Reader.Read (fileinfo.Data(), NumLumps * sizeof(wadlump_t));
Reader.Seek(InfoTableOfs, FileReader::SeekSet);
auto fd = Reader.Read(NumLumps * sizeof(wadlump_t));
auto fileinfo = (const wadlump_t*)fd.data();
Lumps.Resize(NumLumps);
AllocateEntries(NumLumps);
for(uint32_t i = 0; i < NumLumps; i++)
{
// WAD only supports ASCII. It is also the only format which can use valid backslashes in its names.
char n[9];
for(int j = 0; j < 8; j++) n[j] = toupper(fileinfo[i].Name[j]);
n[8] = 0;
// This needs to be done differently. We cannot simply assume that all lumps where the first character's high bit is set are compressed without verification.
// This requires explicit toggling for precisely the files that need it.
#if 0
Lumps[i].Compressed = !(gameinfo.flags & GI_SHAREWARE) && (n[0] & 0x80) == 0x80;
n[0] &= ~0x80;
#else
Lumps[i].Compressed = false;
#endif
Lumps[i].LumpNameSetup(n, stringpool);
Lumps[i].Owner = this;
Lumps[i].Position = isBigEndian ? BigLong(fileinfo[i].FilePos) : LittleLong(fileinfo[i].FilePos);
Lumps[i].LumpSize = isBigEndian ? BigLong(fileinfo[i].Size) : LittleLong(fileinfo[i].Size);
Lumps[i].Namespace = ns_global;
Lumps[i].Flags = Lumps[i].Compressed ? LUMPF_COMPRESSED | LUMPF_SHORTNAME : LUMPF_SHORTNAME;
// Check if the lump is within the WAD file and print a warning if not.
if (Lumps[i].Position + Lumps[i].LumpSize > wadSize || Lumps[i].Position < 0 || Lumps[i].LumpSize < 0)
int ishigh = 0;
for (int j = 0; j < 8; j++)
{
if (Lumps[i].LumpSize != 0)
{
Printf(FSMessageLevel::Warning, "%s: Lump %s contains invalid positioning info and will be ignored\n", FileName, Lumps[i].getName());
Lumps[i].clearName();
}
Lumps[i].LumpSize = Lumps[i].Position = 0;
if (fileinfo[i].Name[j] & 0x80) ishigh |= 1 << j;
n[j] = tolower(fileinfo[i].Name[j]);
}
n[8] = 0;
if (ishigh == 1) n[0] &= 0x7f;
else if (ishigh > 1)
{
// This may not end up printing something proper because we do not know what encoding might have been used.
Printf(FSMessageLevel::Warning, "%s: Lump name %.8s contains invalid characters\n", FileName, fileinfo[i].Name);
}
Entries[i].FileName = nullptr;
Entries[i].Position = isBigEndian ? BigLong(fileinfo[i].FilePos) : LittleLong(fileinfo[i].FilePos);
Entries[i].Length = isBigEndian ? BigLong(fileinfo[i].Size) : LittleLong(fileinfo[i].Size);
Entries[i].Namespace = ns_global;
Entries[i].Flags = ishigh? RESFF_SHORTNAME | RESFF_COMPRESSED : RESFF_SHORTNAME;
Entries[i].Method = ishigh == 1? METHOD_LZSS : METHOD_STORED;
Entries[i].FileName = stringpool->Strdup(n);
// This doesn't set up the namespace yet.
}
GenerateHash(); // Do this before the lump processing below.
SetNamespace("S_START", "S_END", ns_sprites, Printf);
SetNamespace("F_START", "F_END", ns_flats, Printf, true);
SetNamespace("C_START", "C_END", ns_colormaps, Printf);
SetNamespace("A_START", "A_END", ns_acslibrary, Printf);
SetNamespace("TX_START", "TX_END", ns_newtextures, Printf);
SetNamespace("V_START", "V_END", ns_strifevoices, Printf);
SetNamespace("HI_START", "HI_END", ns_hires, Printf);
SetNamespace("VX_START", "VX_END", ns_voxels, Printf);
SetNamespace("s_start", "s_end", ns_sprites, Printf);
SetNamespace("f_start", "f_end", ns_flats, Printf, true);
SetNamespace("c_start", "c_end", ns_colormaps, Printf);
SetNamespace("a_start", "a_end", ns_acslibrary, Printf);
SetNamespace("tx_start", "tx_end", ns_newtextures, Printf);
SetNamespace("v_start", "v_end", ns_strifevoices, Printf);
SetNamespace("hi_start", "hi_end", ns_hires, Printf);
SetNamespace("vx_start", "vx_end", ns_voxels, Printf);
SkinHack(Printf);
return true;
}
@ -251,10 +182,10 @@ bool FWadFile::Open(LumpFilterInfo*, FileSystemMessageFunc Printf)
inline bool FWadFile::IsMarker(int lump, const char *marker)
{
if (Lumps[lump].getName()[0] == marker[0])
if (Entries[lump].FileName[0] == marker[0])
{
return (!strcmp(Lumps[lump].getName(), marker) ||
(marker[1] == '_' && !strcmp(Lumps[lump].getName() +1, marker)));
return (!strcmp(Entries[lump].FileName, marker) ||
(marker[1] == '_' && !strcmp(Entries[lump].FileName +1, marker)));
}
else return false;
}
@ -283,20 +214,20 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
bool warned = false;
int numstartmarkers = 0, numendmarkers = 0;
unsigned int i;
TArray<Marker> markers;
std::vector<Marker> markers;
for(i = 0; i < NumLumps; i++)
{
if (IsMarker(i, startmarker))
{
Marker m = { 0, i };
markers.Push(m);
markers.push_back(m);
numstartmarkers++;
}
else if (IsMarker(i, endmarker))
{
Marker m = { 1, i };
markers.Push(m);
markers.push_back(m);
numendmarkers++;
}
}
@ -312,15 +243,15 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
{
// We have found no F_START but one or more F_END markers.
// mark all lumps before the last F_END marker as potential flats.
unsigned int end = markers[markers.Size()-1].index;
unsigned int end = markers[markers.size()-1].index;
for(unsigned int ii = 0; ii < end; ii++)
{
if (Lumps[ii].LumpSize == 4096)
if (Entries[ii].Length == 4096)
{
// We can't add this to the flats namespace but
// it needs to be flagged for the texture manager.
Printf(FSMessageLevel::DebugNotify, "%s: Marking %s as potential flat\n", FileName, Lumps[ii].getName());
Lumps[ii].Flags |= LUMPF_MAYBEFLAT;
Printf(FSMessageLevel::DebugNotify, "%s: Marking %s as potential flat\n", FileName, Entries[ii].FileName);
Entries[ii].Flags |= RESFF_MAYBEFLAT;
}
}
}
@ -328,7 +259,7 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
}
i = 0;
while (i < markers.Size())
while (i < markers.size())
{
int start, end;
if (markers[i].markertype != 0)
@ -340,21 +271,21 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
start = i++;
// skip over subsequent x_START markers
while (i < markers.Size() && markers[i].markertype == 0)
while (i < markers.size() && markers[i].markertype == 0)
{
Printf(FSMessageLevel::Warning, "%s: duplicate %s marker found.\n", FileName, startmarker);
i++;
continue;
}
// same for x_END markers
while (i < markers.Size()-1 && (markers[i].markertype == 1 && markers[i+1].markertype == 1))
while (i < markers.size()-1 && (markers[i].markertype == 1 && markers[i+1].markertype == 1))
{
Printf(FSMessageLevel::Warning, "%s: duplicate %s marker found.\n", FileName, endmarker);
i++;
continue;
}
// We found a starting marker but no end marker. Ignore this block.
if (i >= markers.Size())
if (i >= markers.size())
{
Printf(FSMessageLevel::Warning, "%s: %s marker without corresponding %s found.\n", FileName, startmarker, endmarker);
end = NumLumps;
@ -368,7 +299,7 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
Printf(FSMessageLevel::DebugNotify, "%s: Found %s block at (%d-%d)\n", FileName, startmarker, markers[start].index, end);
for(int j = markers[start].index + 1; j < end; j++)
{
if (Lumps[j].Namespace != ns_global)
if (Entries[j].Namespace != ns_global)
{
if (!warned)
{
@ -376,17 +307,17 @@ void FWadFile::SetNamespace(const char *startmarker, const char *endmarker, name
}
warned = true;
}
else if (space == ns_sprites && Lumps[j].LumpSize < 8)
else if (space == ns_sprites && Entries[j].Length < 8)
{
// sf 26/10/99:
// ignore sprite lumps smaller than 8 bytes (the smallest possible)
// in size -- this was used by some dmadds wads
// as an 'empty' graphics resource
Printf(FSMessageLevel::DebugWarn, "%s: Skipped empty sprite %s (lump %d)\n", FileName, Lumps[j].getName(), j);
Printf(FSMessageLevel::DebugWarn, "%s: Skipped empty sprite %s (lump %d)\n", FileName, Entries[j].FileName, j);
}
else
{
Lumps[j].Namespace = space;
Entries[j].Namespace = space;
}
}
}
@ -416,11 +347,11 @@ void FWadFile::SkinHack (FileSystemMessageFunc Printf)
for (i = 0; i < NumLumps; i++)
{
FResourceLump *lump = &Lumps[i];
auto lump = &Entries[i];
if (!strnicmp(lump->getName(), "S_SKIN", 6))
if (!strnicmp(lump->FileName, "S_SKIN", 6))
{ // Wad has at least one skin.
lump->LumpNameSetup("S_SKIN", nullptr);
lump->FileName = "S_SKIN";
if (!skinned)
{
skinned = true;
@ -428,24 +359,24 @@ void FWadFile::SkinHack (FileSystemMessageFunc Printf)
for (j = 0; j < NumLumps; j++)
{
Lumps[j].Namespace = namespc;
Entries[j].Namespace = namespc;
}
namespc++;
}
}
// needless to say, this check is entirely useless these days as map names can be more diverse..
if ((lump->getName()[0] == 'M' &&
lump->getName()[1] == 'A' &&
lump->getName()[2] == 'P' &&
lump->getName()[3] >= '0' && lump->getName()[3] <= '9' &&
lump->getName()[4] >= '0' && lump->getName()[4] <= '9' &&
lump->getName()[5] == '\0')
if ((lump->FileName[0] == 'M' &&
lump->FileName[1] == 'A' &&
lump->FileName[2] == 'P' &&
lump->FileName[3] >= '0' && lump->FileName[3] <= '9' &&
lump->FileName[4] >= '0' && lump->FileName[4] <= '9' &&
lump->FileName[5] == '\0')
||
(lump->getName()[0] == 'E' &&
lump->getName()[1] >= '0' && lump->getName()[1] <= '9' &&
lump->getName()[2] == 'M' &&
lump->getName()[3] >= '0' && lump->getName()[3] <= '9' &&
lump->getName()[4] == '\0'))
(lump->FileName[0] == 'E' &&
lump->FileName[1] >= '0' && lump->FileName[1] <= '9' &&
lump->FileName[2] == 'M' &&
lump->FileName[3] >= '0' && lump->FileName[3] <= '9' &&
lump->FileName[4] == '\0'))
{
hasmap = true;
}

77
source/common/filesystem/source/file_whres.cpp
View file

@ -34,76 +34,61 @@
**
*/
#include "resourcefile_internal.h"
#include "resourcefile.h"
#include "fs_stringpool.h"
#include "fs_swap.h"
namespace FileSys {
using namespace byteswap;
//==========================================================================
//
// WH resource file
//
//==========================================================================
class FWHResFile : public FUncompressedFile
{
const char* BaseName;
public:
FWHResFile(const char * filename, FileReader &file, StringPool* sp);
bool Open(LumpFilterInfo* filter);
};
//==========================================================================
//
//
//
//==========================================================================
FWHResFile::FWHResFile(const char *filename, FileReader &file, StringPool* sp)
: FUncompressedFile(filename, file, sp)
{
BaseName = stringpool->Strdup(ExtractBaseName(filename, false).c_str());
}
//==========================================================================
//
// Open it
//
//==========================================================================
bool FWHResFile::Open(LumpFilterInfo*)
bool OpenWHRes(FResourceFile* file, LumpFilterInfo*)
{
uint32_t directory[1024];
Reader.Seek(-4096, FileReader::SeekEnd);
Reader.Read(directory, 4096);
auto BaseName = ExtractBaseName(file->GetFileName());
auto Reader = file->GetContainerReader();
Reader->Seek(-4096, FileReader::SeekEnd);
Reader->Read(directory, 4096);
int nl =1024/3;
Lumps.Resize(nl);
int k;
for (k = 0; k < nl; k++)
{
uint32_t offset = LittleLong(directory[k * 3]) * 4096;
uint32_t length = LittleLong(directory[k * 3 + 1]);
if (length == 0)
{
break;
}
}
auto Entries = file->AllocateEntries(k);
auto NumLumps = k;
int i = 0;
for(int k = 0; k < nl; k++)
for(k = 0; k < NumLumps; k++)
{
uint32_t offset = LittleLong(directory[k*3]) * 4096;
uint32_t length = LittleLong(directory[k*3+1]);
if (length == 0) break;
char num[6];
snprintf(num, 6, "/%04d", k);
std::string synthname = BaseName;
synthname += num;
Lumps[i].LumpNameSetup(synthname.c_str(), stringpool);
Lumps[i].Owner = this;
Lumps[i].Position = offset;
Lumps[i].LumpSize = length;
std::string synthname = BaseName + num;
Entries[i].Position = offset;
Entries[i].Length = length;
Entries[i].Flags = RESFF_FULLPATH;
Entries[i].Namespace = ns_global;
Entries[i].ResourceID = -1;
Entries[i].Method = METHOD_STORED;
Entries[i].FileName = file->NormalizeFileName(synthname.c_str());
i++;
}
NumLumps = i;
Lumps.Clamp(NumLumps);
Lumps.ShrinkToFit();
return true;
}
@ -134,8 +119,8 @@ FResourceFile *CheckWHRes(const char *filename, FileReader &file, LumpFilterInfo
if (offset != checkpos || length == 0 || offset + length >= (size_t)size - 4096 ) return nullptr;
checkpos += (length+4095) / 4096;
}
auto rf = new FWHResFile(filename, file, sp);
if (rf->Open(filter)) return rf;
auto rf = new FResourceFile(filename, file, sp);
if (OpenWHRes(rf, filter)) return rf;
file = rf->Destroy();
}
return NULL;

449
source/common/filesystem/source/file_zip.cpp
View file

@ -3,7 +3,7 @@
**
**---------------------------------------------------------------------------
** Copyright 1998-2009 Randy Heit
** Copyright 2005-2009 Christoph Oelckers
** Copyright 2005-2023 Christoph Oelckers
** All rights reserved.
**
** Redistribution and use in source and binary forms, with or without
@ -35,78 +35,18 @@
#include <time.h>
#include <stdexcept>
#include "file_zip.h"
#include "w_zip.h"
#include "ancientzip.h"
#include "resourcefile.h"
#include "fs_findfile.h"
#include "fs_swap.h"
#include "fs_stringpool.h"
namespace FileSys {
using namespace byteswap;
#define BUFREADCOMMENT (0x400)
//==========================================================================
//
// Decompression subroutine
//
//==========================================================================
static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdiff_t LumpSize, ptrdiff_t CompressedSize, int GPFlags, bool exceptions)
{
switch (Method)
{
case METHOD_STORED:
{
Reader.Read(Cache, LumpSize);
break;
}
case METHOD_DEFLATE:
case METHOD_BZIP2:
case METHOD_LZMA:
case METHOD_XZ:
{
FileReader frz;
if (frz.OpenDecompressor(Reader, LumpSize, Method, false, exceptions))
{
frz.Read(Cache, LumpSize);
}
break;
}
// Fixme: These should also use a stream
case METHOD_IMPLODE:
{
FZipExploder exploder;
if (exploder.Explode((unsigned char*)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize, GPFlags) == -1)
{
// decompression failed so zero the cache.
memset(Cache, 0, LumpSize);
}
break;
}
case METHOD_SHRINK:
{
ShrinkLoop((unsigned char *)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize);
break;
}
default:
assert(0);
return false;
}
return true;
}
bool FCompressedBuffer::Decompress(char *destbuffer)
{
FileReader mr;
mr.OpenMemory(mBuffer, mCompressedSize);
return UncompressZipLump(destbuffer, mr, mMethod, mSize, mCompressedSize, mZipFlags, false);
}
//-----------------------------------------------------------------------
//
// Finds the central directory end record in the end of the file.
@ -166,10 +106,25 @@ static uint32_t Zip_FindCentralDir(FileReader &fin, bool* zip64)
//
//==========================================================================
class FZipFile : public FResourceFile
{
void SetEntryAddress(uint32_t entry) override;
public:
FZipFile(const char* filename, FileReader& file, StringPool* sp);
bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf);
FCompressedBuffer GetRawData(uint32_t entry) override;
};
//==========================================================================
//
// Zip file
//
//==========================================================================
FZipFile::FZipFile(const char * filename, FileReader &file, StringPool* sp)
: FResourceFile(filename, file, sp)
{
Lumps = NULL;
}
bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
@ -178,8 +133,6 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
uint32_t centraldir = Zip_FindCentralDir(Reader, &zip64);
int skipped = 0;
Lumps = NULL;
if (centraldir == 0)
{
Printf(FSMessageLevel::Error, "%s: ZIP file corrupt!\n", FileName);
@ -225,15 +178,12 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
dirsize = info.DirectorySize;
DirectoryOffset = info.DirectoryOffset;
}
Lumps = new FZipLump[NumLumps];
// Load the entire central directory. Too bad that this contains variable length entries...
void *directory = malloc(dirsize);
Reader.Seek(DirectoryOffset, FileReader::SeekSet);
Reader.Read(directory, dirsize);
char *dirptr = (char*)directory;
FZipLump *lump_p = Lumps;
std::string name0, name1;
bool foundspeciallump = false;
@ -259,58 +209,11 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
Printf(FSMessageLevel::Error, "%s: Central directory corrupted.", FileName);
return false;
}
for (auto& c : name) c = tolower(c);
auto vv = name.find("__macosx");
if (name.find("filter/") == 0)
continue; // 'filter' is a reserved name of the file system.
if (name.find("__macosx") == 0)
continue; // skip Apple garbage. At this stage only the root folder matters.
if (name.find(".bat") != std::string::npos || name.find(".exe") != std::string::npos)
continue; // also ignore executables for this.
if (!foundprefix)
{
// check for special names, if one of these gets found this must be treated as a normal zip.
bool isspecial = name.find("/") == std::string::npos ||
(filter && std::find(filter->reservedFolders.begin(), filter->reservedFolders.end(), name) != filter->reservedFolders.end());
if (isspecial) break;
name0 = std::string(name, 0, name.rfind("/")+1);
name1 = std::string(name, 0, name.find("/") + 1);
foundprefix = true;
}
if (name.find(name0) != 0)
{
if (!name1.empty())
{
name0 = name1;
if (name.find(name0) != 0)
{
name0 = "";
}
}
if (name0.empty())
break;
}
if (!foundspeciallump && filter)
{
// at least one of the more common definition lumps must be present.
for (auto &p : filter->requiredPrefixes)
{
if (name.find(name0 + p) == 0 || name.rfind(p) == size_t(name.length() - p.length()))
{
foundspeciallump = true;
break;
}
}
}
}
// If it ran through the list without finding anything it should not attempt any path remapping.
if (!foundspeciallump) name0 = "";
dirptr = (char*)directory;
lump_p = Lumps;
AllocateEntries(NumLumps);
auto Entry = Entries;
for (uint32_t i = 0; i < NumLumps; i++)
{
FZipCentralDirectoryInfo *zip_fh = (FZipCentralDirectoryInfo *)dirptr;
@ -329,13 +232,6 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
return false;
}
if (name.find("__macosx") == 0 || name.find("__MACOSX") == 0)
{
skipped++;
continue; // Weed out Apple's resource fork garbage right here because it interferes with safe operation.
}
if (!name0.empty()) name = std::string(name, name0.length());
// skip Directories
if (name.empty() || (name.back() == '/' && LittleLong(zip_fh->UncompressedSize32) == 0))
{
@ -366,9 +262,6 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
continue;
}
FixPathSeparator(&name.front());
for (auto& c : name) c = tolower(c);
uint32_t UncompressedSize =LittleLong(zip_fh->UncompressedSize32);
uint32_t CompressedSize = LittleLong(zip_fh->CompressedSize32);
uint64_t LocalHeaderOffset = LittleLong(zip_fh->LocalHeaderOffset32);
@ -399,54 +292,59 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf)
}
}
lump_p->LumpNameSetup(name.c_str(), stringpool);
lump_p->LumpSize = UncompressedSize;
lump_p->Owner = this;
Entry->FileName = NormalizeFileName(name.c_str());
Entry->Length = UncompressedSize;
// The start of the Reader will be determined the first time it is accessed.
lump_p->Flags = LUMPF_FULLPATH;
lump_p->NeedFileStart = true;
lump_p->Method = uint8_t(zip_fh->Method);
if (lump_p->Method != METHOD_STORED) lump_p->Flags |= LUMPF_COMPRESSED;
lump_p->GPFlags = zip_fh->Flags;
lump_p->CRC32 = zip_fh->CRC32;
lump_p->CompressedSize = CompressedSize;
lump_p->Position = LocalHeaderOffset;
lump_p->CheckEmbedded(filter);
Entry->Flags = RESFF_FULLPATH | RESFF_NEEDFILESTART;
Entry->Method = uint8_t(zip_fh->Method);
if (Entry->Method != METHOD_STORED) Entry->Flags |= RESFF_COMPRESSED;
if (Entry->Method == METHOD_IMPLODE)
{
// for Implode merge the flags into the compression method to make handling in the file system easier and save one variable.
if ((zip_fh->Flags & 6) == 2) Entry->Method = METHOD_IMPLODE_2;
else if ((zip_fh->Flags & 6) == 4) Entry->Method = METHOD_IMPLODE_4;
else if ((zip_fh->Flags & 6) == 6) Entry->Method = METHOD_IMPLODE_6;
else Entry->Method = METHOD_IMPLODE_0;
}
Entry->CRC32 = zip_fh->CRC32;
Entry->CompressedSize = CompressedSize;
Entry->Position = LocalHeaderOffset;
Entry++;
lump_p++;
}
// Resize the lump record array to its actual size
NumLumps -= skipped;
free(directory);
GenerateHash();
PostProcessArchive(&Lumps[0], sizeof(FZipLump), filter);
PostProcessArchive(filter);
return true;
}
//==========================================================================
//
// Zip file
//
//
//==========================================================================
FZipFile::~FZipFile()
FCompressedBuffer FZipFile::GetRawData(uint32_t entry)
{
if (Lumps != NULL) delete [] Lumps;
}
FCompressedBuffer cbuf;
//==========================================================================
//
//
//
//==========================================================================
FCompressedBuffer FZipLump::GetRawData()
{
FCompressedBuffer cbuf = { (unsigned)LumpSize, (unsigned)CompressedSize, Method, GPFlags, CRC32, new char[CompressedSize] };
if (NeedFileStart) SetLumpAddress();
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
Owner->GetContainerReader()->Read(cbuf.mBuffer, CompressedSize);
if (entry >= NumLumps >> Entries[entry].Length == 0)
{
cbuf = { 0, 0, METHOD_STORED, 0, 0, nullptr };
}
else
{
auto& e = Entries[entry];
cbuf = { e.Length, e.CompressedSize, e.Method, e.CRC32, new char[e.CompressedSize] };
if (e.Flags & RESFF_NEEDFILESTART) SetEntryAddress(entry);
Reader.Seek(e.Position, FileReader::SeekSet);
Reader.Read(cbuf.mBuffer, e.CompressedSize);
}
return cbuf;
}
@ -456,7 +354,7 @@ FCompressedBuffer FZipLump::GetRawData()
//
//==========================================================================
void FZipLump::SetLumpAddress()
void FZipFile::SetEntryAddress(uint32_t entry)
{
// This file is inside a zip and has not been opened before.
// Position points to the start of the local file header, which we must
@ -464,69 +362,11 @@ void FZipLump::SetLumpAddress()
FZipLocalFileHeader localHeader;
int skiplen;
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
Owner->GetContainerReader()->Read(&localHeader, sizeof(localHeader));
Reader.Seek(Entries[entry].Position, FileReader::SeekSet);
Reader.Read(&localHeader, sizeof(localHeader));
skiplen = LittleShort(localHeader.NameLength) + LittleShort(localHeader.ExtraLength);
Position += sizeof(localHeader) + skiplen;
NeedFileStart = false;
}
//==========================================================================
//
// Get reader (only returns non-NULL if not encrypted)
//
//==========================================================================
FileReader *FZipLump::GetReader()
{
// Don't return the reader if this lump is encrypted
// In that case always force caching of the lump
if (Method == METHOD_STORED)
{
if (NeedFileStart) SetLumpAddress();
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
return Owner->GetContainerReader();
}
else return NULL;
}
//==========================================================================
//
// Fills the lump cache and performs decompression
//
//==========================================================================
int FZipLump::FillCache()
{
if (NeedFileStart) SetLumpAddress();
const char *buffer;
if (Method == METHOD_STORED && (buffer = Owner->GetContainerReader()->GetBuffer()) != NULL)
{
// This is an in-memory file so the cache can point directly to the file's data.
Cache = const_cast<char*>(buffer) + Position;
RefCount = -1;
return -1;
}
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
Cache = new char[LumpSize];
UncompressZipLump(Cache, *Owner->GetContainerReader(), Method, LumpSize, CompressedSize, GPFlags, true);
RefCount = 1;
return 1;
}
//==========================================================================
//
//
//
//==========================================================================
int FZipLump::GetFileOffset()
{
if (Method != METHOD_STORED) return -1;
if (NeedFileStart) SetLumpAddress();
return (int)Position;
Entries[entry].Position += sizeof(localHeader) + skiplen;
Entries[entry].Flags &= ~RESFF_NEEDFILESTART;
}
//==========================================================================
@ -555,171 +395,4 @@ FResourceFile *CheckZip(const char *filename, FileReader &file, LumpFilterInfo*
}
//==========================================================================
//
// time_to_dos
//
// Converts time from struct tm to the DOS format used by zip files.
//
//==========================================================================
static std::pair<uint16_t, uint16_t> time_to_dos(struct tm *time)
{
std::pair<uint16_t, uint16_t> val;
if (time == NULL || time->tm_year < 80)
{
val.first = val.second = 0;
}
else
{
val.first = (time->tm_year - 80) * 512 + (time->tm_mon + 1) * 32 + time->tm_mday;
val.second= time->tm_hour * 2048 + time->tm_min * 32 + time->tm_sec / 2;
}
return val;
}
//==========================================================================
//
// append_to_zip
//
// Write a given file to the zipFile.
//
// zipfile: zip object to be written to
//
// returns: position = success, -1 = error
//
//==========================================================================
static int AppendToZip(FileWriter *zip_file, const FCompressedBuffer &content, std::pair<uint16_t, uint16_t> &dostime)
{
FZipLocalFileHeader local;
int position;
local.Magic = ZIP_LOCALFILE;
local.VersionToExtract[0] = 20;
local.VersionToExtract[1] = 0;
local.Flags = content.mMethod == METHOD_DEFLATE ? LittleShort((uint16_t)2) : LittleShort((uint16_t)content.mZipFlags);
local.Method = LittleShort((uint16_t)content.mMethod);
local.ModDate = LittleShort(dostime.first);
local.ModTime = LittleShort(dostime.second);
local.CRC32 = content.mCRC32;
local.UncompressedSize = LittleLong((unsigned)content.mSize);
local.CompressedSize = LittleLong((unsigned)content.mCompressedSize);
local.NameLength = LittleShort((unsigned short)strlen(content.filename));
local.ExtraLength = 0;
// Fill in local directory header.
position = (int)zip_file->Tell();
// Write out the header, file name, and file data.
if (zip_file->Write(&local, sizeof(local)) != sizeof(local) ||
zip_file->Write(content.filename, strlen(content.filename)) != strlen(content.filename) ||
zip_file->Write(content.mBuffer, content.mCompressedSize) != content.mCompressedSize)
{
return -1;
}
return position;
}
//==========================================================================
//
// write_central_dir
//
// Writes the central directory entry for a file.
//
//==========================================================================
int AppendCentralDirectory(FileWriter *zip_file, const FCompressedBuffer &content, std::pair<uint16_t, uint16_t> &dostime, int position)
{
FZipCentralDirectoryInfo dir;
dir.Magic = ZIP_CENTRALFILE;
dir.VersionMadeBy[0] = 20;
dir.VersionMadeBy[1] = 0;
dir.VersionToExtract[0] = 20;
dir.VersionToExtract[1] = 0;
dir.Flags = content.mMethod == METHOD_DEFLATE ? LittleShort((uint16_t)2) : LittleShort((uint16_t)content.mZipFlags);
dir.Method = LittleShort((uint16_t)content.mMethod);
dir.ModTime = LittleShort(dostime.first);
dir.ModDate = LittleShort(dostime.second);
dir.CRC32 = content.mCRC32;
dir.CompressedSize32 = LittleLong((unsigned)content.mCompressedSize);
dir.UncompressedSize32 = LittleLong((unsigned)content.mSize);
dir.NameLength = LittleShort((unsigned short)strlen(content.filename));
dir.ExtraLength = 0;
dir.CommentLength = 0;
dir.StartingDiskNumber = 0;
dir.InternalAttributes = 0;
dir.ExternalAttributes = 0;
dir.LocalHeaderOffset32 = LittleLong((unsigned)position);
if (zip_file->Write(&dir, sizeof(dir)) != sizeof(dir) ||
zip_file->Write(content.filename, strlen(content.filename)) != strlen(content.filename))
{
return -1;
}
return 0;
}
bool WriteZip(const char* filename, const FCompressedBuffer* content, size_t contentcount)
{
// try to determine local time
struct tm *ltime;
time_t ttime;
ttime = time(nullptr);
ltime = localtime(&ttime);
auto dostime = time_to_dos(ltime);
TArray<int> positions;
auto f = FileWriter::Open(filename);
if (f != nullptr)
{
for (size_t i = 0; i < contentcount; i++)
{
int pos = AppendToZip(f, content[i], dostime);
if (pos == -1)
{
delete f;
remove(filename);
return false;
}
positions.Push(pos);
}
int dirofs = (int)f->Tell();
for (size_t i = 0; i < contentcount; i++)
{
if (AppendCentralDirectory(f, content[i], dostime, positions[i]) < 0)
{
delete f;
remove(filename);
return false;
}
}
// Write the directory terminator.
FZipEndOfCentralDirectory dirend;
dirend.Magic = ZIP_ENDOFDIR;
dirend.DiskNumber = 0;
dirend.FirstDisk = 0;
dirend.NumEntriesOnAllDisks = dirend.NumEntries = LittleShort((uint16_t)contentcount);
dirend.DirectoryOffset = LittleLong((unsigned)dirofs);
dirend.DirectorySize = LittleLong((uint32_t)(f->Tell() - dirofs));
dirend.ZipCommentLength = 0;
if (f->Write(&dirend, sizeof(dirend)) != sizeof(dirend))
{
delete f;
remove(filename);
return false;
}
delete f;
return true;
}
return false;
}
}

51
source/common/filesystem/source/file_zip.h
View file

@ -1,51 +0,0 @@
#ifndef __FILE_ZIP_H
#define __FILE_ZIP_H
#include "resourcefile.h"
namespace FileSys {
//==========================================================================
//
// Zip Lump
//
//==========================================================================
struct FZipLump : public FResourceLump
{
uint16_t GPFlags;
uint8_t Method;
bool NeedFileStart;
int CompressedSize;
int64_t Position;
unsigned CRC32;
virtual FileReader *GetReader();
virtual int FillCache() override;
private:
void SetLumpAddress();
virtual int GetFileOffset();
FCompressedBuffer GetRawData();
};
//==========================================================================
//
// Zip file
//
//==========================================================================
class FZipFile : public FResourceFile
{
FZipLump *Lumps;
public:
FZipFile(const char * filename, FileReader &file, StringPool* sp);
virtual ~FZipFile();
bool Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf);
virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; }
};
}
#endif

62
source/common/filesystem/source/files.cpp
View file

@ -35,6 +35,9 @@
#include <string>
#include <memory>
#include <algorithm>
#include <assert.h>
#include <string.h>
#include "files_internal.h"
namespace FileSys {
@ -326,15 +329,15 @@ char *MemoryReader::Gets(char *strbuf, ptrdiff_t len)
int BufferingReader::FillBuffer(ptrdiff_t newpos)
{
if (newpos > Length) newpos = Length;
if (newpos < bufferpos) return 0;
auto read = baseReader->Read(&buf[bufferpos], newpos - bufferpos);
if (newpos <= bufferpos) return 0;
auto read = baseReader->Read(&buf.writable()[bufferpos], newpos - bufferpos);
bufferpos += read;
if (bufferpos == Length)
{
// we have read the entire file, so delete our data provider.
baseReader.reset();
}
return read == newpos - bufferpos ? 0 : -1;
return newpos == bufferpos ? 0 : -1;
}
ptrdiff_t BufferingReader::Seek(ptrdiff_t offset, int origin)
@ -392,45 +395,40 @@ bool FileReader::OpenMemory(const void *mem, FileReader::Size length)
return true;
}
bool FileReader::OpenMemoryArray(const void *mem, FileReader::Size length)
bool FileReader::OpenMemoryArray(FileData& data)
{
Close();
mReader = new MemoryArrayReader<std::vector<uint8_t>>((const char *)mem, length);
if (data.size() > 0) mReader = new MemoryArrayReader(data);
return true;
}
bool FileReader::OpenMemoryArray(std::vector<uint8_t>& data)
FileData FileReader::Read(size_t len)
{
Close();
if (data.size() > 0) mReader = new MemoryArrayReader<std::vector<uint8_t>>(data);
return true;
}
bool FileReader::OpenMemoryArray(ResourceData& data)
{
Close();
if (data.size() > 0) mReader = new MemoryArrayReader<ResourceData>(data);
return true;
}
bool FileReader::OpenMemoryArray(std::function<bool(std::vector<uint8_t>&)> getter)
{
auto reader = new MemoryArrayReader<std::vector<uint8_t>>(nullptr, 0);
if (getter(reader->GetArray()))
FileData buffer;
if (len > 0)
{
Close();
reader->UpdateBuffer();
mReader = reader;
return true;
}
else
{
// This will keep the old buffer, if one existed
delete reader;
return false;
Size length = mReader->Read(buffer.allocate(len), len);
if ((size_t)length < len) buffer.allocate(length);
}
return buffer;
}
FileData FileReader::ReadPadded(size_t padding)
{
auto len = GetLength();
FileData buffer;
if (len > 0)
{
auto p = (char*)buffer.allocate(len + padding);
Size length = mReader->Read(p, len);
if (length < len) buffer.clear();
else memset(p + len, 0, padding);
}
return buffer;
}
//==========================================================================
//

129
source/common/filesystem/source/files_decompress.cpp
View file

@ -45,6 +45,9 @@
#include <stdexcept>
#include "fs_files.h"
#include "files_internal.h"
#include "ancientzip.h"
#include "fs_decompress.h"
namespace FileSys {
using namespace byteswap;
@ -53,6 +56,7 @@ namespace FileSys {
class DecompressorBase : public FileReaderInterface
{
bool exceptions = false;
public:
// These do not work but need to be defined to satisfy the FileReaderInterface.
// They will just error out when called.
@ -64,6 +68,10 @@ public:
void EnableExceptions(bool on) { exceptions = on; }
protected:
DecompressorBase()
{
//seekable = false;
}
FileReader* File = nullptr;
FileReader OwnedFile;
};
@ -174,7 +182,7 @@ public:
inflateEnd (&Stream);
}
ptrdiff_t Read (void *buffer, ptrdiff_t len) override
ptrdiff_t Read (void *buffer, ptrdiff_t olen) override
{
int err = 0;
@ -183,6 +191,7 @@ public:
DecompressionError("File not open");
return 0;
}
auto len = olen;
if (len == 0) return 0;
while (len > 0)
@ -215,7 +224,7 @@ public:
return 0;
}
return len - Stream.avail_out;
return olen - Stream.avail_out;
}
void FillBuffer ()
@ -841,19 +850,22 @@ public:
};
bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions)
bool OpenDecompressor(FileReader& self, FileReader &parent, FileReader::Size length, int method, int flags)
{
FileReaderInterface* fr = nullptr;
DecompressorBase* dec = nullptr;
try
{
FileReader* p = &parent;
switch (method & ~METHOD_TRANSFEROWNER)
bool exceptions = !!(flags & DCF_EXCEPTIONS);
switch (method)
{
case METHOD_DEFLATE:
case METHOD_ZLIB:
{
auto idec = new DecompressorZ;
dec = idec;
fr = dec = idec;
idec->EnableExceptions(exceptions);
if (!idec->Open(p, method == METHOD_DEFLATE))
{
@ -865,7 +877,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
case METHOD_BZIP2:
{
auto idec = new DecompressorBZ2;
dec = idec;
fr = dec = idec;
idec->EnableExceptions(exceptions);
if (!idec->Open(p))
{
@ -877,7 +889,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
case METHOD_LZMA:
{
auto idec = new DecompressorLZMA;
dec = idec;
fr = dec = idec;
idec->EnableExceptions(exceptions);
if (!idec->Open(p, length))
{
@ -889,7 +901,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
case METHOD_XZ:
{
auto idec = new DecompressorXZ;
dec = idec;
fr = dec = idec;
idec->EnableExceptions(exceptions);
if (!idec->Open(p, length))
{
@ -901,7 +913,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
case METHOD_LZSS:
{
auto idec = new DecompressorLZSS;
dec = idec;
fr = dec = idec;
idec->EnableExceptions(exceptions);
if (!idec->Open(p))
{
@ -911,34 +923,103 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
break;
}
// todo: METHOD_IMPLODE, METHOD_SHRINK
// The decoders for these legacy formats can only handle the full data in one go so we have to perform the entire decompression here.
case METHOD_IMPLODE_0:
case METHOD_IMPLODE_2:
case METHOD_IMPLODE_4:
case METHOD_IMPLODE_6:
{
FileData buffer(nullptr, length);
FZipExploder exploder;
if (exploder.Explode(buffer.writable(), length, *p, p->GetLength(), method - METHOD_IMPLODE_MIN) == -1)
{
if (exceptions)
{
throw FileSystemException("DecompressImplode failed");
}
return false;
}
fr = new MemoryArrayReader(buffer);
flags &= ~(DCF_SEEKABLE | DCF_CACHED);
break;
}
case METHOD_SHRINK:
{
FileData buffer(nullptr, length);
ShrinkLoop(buffer.writable(), length, *p, p->GetLength()); // this never fails.
fr = new MemoryArrayReader(buffer);
flags &= ~(DCF_SEEKABLE | DCF_CACHED);
break;
}
// While this could be made a buffering reader it isn't worth the effort because only stock RFFs are encrypted and they do not contain large files.
case METHOD_RFFCRYPT:
{
FileData buffer = p->Read(length);
auto bufr = buffer.writable();
FileReader::Size cryptlen = std::min<FileReader::Size>(length, 256);
for (FileReader::Size i = 0; i < cryptlen; ++i)
{
bufr[i] ^= i >> 1;
}
fr = new MemoryArrayReader(buffer);
flags &= ~(DCF_SEEKABLE | DCF_CACHED);
break;
}
default:
return false;
}
if (method & METHOD_TRANSFEROWNER)
if (dec)
{
dec->SetOwnsReader();
if (flags & DCF_TRANSFEROWNER)
{
dec->SetOwnsReader();
}
dec->Length = length;
}
dec->Length = length;
if (!seekable)
if ((flags & DCF_CACHED))
{
Close();
mReader = dec;
return true;
// read everything into a MemoryArrayReader.
FileData data(nullptr, length);
fr->Read(data.writable(), length);
fr = new MemoryArrayReader(data);
}
else
else if ((flags & DCF_SEEKABLE))
{
// todo: create a wrapper. for now this fails
delete dec;
return false;
// create a wrapper that can buffer the content so that seeking is possible
fr = new BufferingReader(fr);
}
}
self = FileReader(fr);
return true;
}
catch (...)
{
if (dec) delete dec;
if (fr) delete fr;
throw;
}
}
bool FCompressedBuffer::Decompress(char* destbuffer)
{
if (mMethod == METHOD_STORED)
{
memcpy(destbuffer, mBuffer, mSize);
return true;
}
else
{
FileReader mr;
mr.OpenMemory(mBuffer, mCompressedSize);
FileReader frz;
if (OpenDecompressor(frz, mr, mSize, mMethod))
{
return frz.Read(destbuffer, mSize) != mSize;
}
}
return false;
}
}

23
source/common/filesystem/source/files_internal.h
View file

@ -31,7 +31,7 @@ public:
class BufferingReader : public MemoryReader
{
std::vector<uint8_t> buf;
FileData buf;
std::unique_ptr<FileReaderInterface> baseReader;
ptrdiff_t bufferpos = 0;
@ -40,6 +40,9 @@ public:
BufferingReader(FileReaderInterface* base)
: baseReader(base)
{
Length = base->Length;
buf.allocate(Length);
bufptr = (const char*)buf.data();
}
ptrdiff_t Seek(ptrdiff_t offset, int origin) override;
@ -55,10 +58,9 @@ public:
//
//==========================================================================
template<class T>
class MemoryArrayReader : public MemoryReader
{
T buf;
FileData buf;
public:
MemoryArrayReader()
@ -67,24 +69,18 @@ public:
Length = 0;
}
MemoryArrayReader(const char* buffer, ptrdiff_t length)
MemoryArrayReader(size_t len)
{
if (length > 0)
{
buf.resize(length);
memcpy(&buf[0], buffer, length);
}
buf.allocate(len);
UpdateBuffer();
}
MemoryArrayReader(T& buffer)
MemoryArrayReader(FileData& buffer)
{
buf = std::move(buffer);
UpdateBuffer();
}
std::vector<uint8_t>& GetArray() { return buf; }
void UpdateBuffer()
{
bufptr = (const char*)buf.data();
@ -93,7 +89,4 @@ public:
}
};
bool OpenMemoryArray(std::vector<uint8_t>& data); // take the given array
}

177
source/common/filesystem/source/filesystem.cpp
View file

@ -41,7 +41,7 @@
#include <ctype.h>
#include <string.h>
#include "resourcefile_internal.h"
#include "resourcefile.h"
#include "fs_filesystem.h"
#include "fs_findfile.h"
#include "md5.hpp"
@ -90,29 +90,37 @@ static void md5Hash(FileReader& reader, uint8_t* digest)
struct FileSystem::LumpRecord
{
FResourceLump *lump;
FResourceFile *resfile;
LumpShortName shortName;
const char* LongName;
int rfnum;
int Namespace;
int resindex;
int16_t rfnum; // this is not necessarily the same as resfile's index!
int16_t Namespace;
int resourceId;
int flags;
void SetFromLump(int filenum, FResourceLump* lmp, StringPool* sp)
void SetFromLump(FResourceFile* file, int fileindex, int filenum, StringPool* sp, const char* name = nullptr)
{
lump = lmp;
if (fileindex == 649 && filenum == 0)
{
int a = 0;
}
resfile = file;
resindex = fileindex;
rfnum = filenum;
flags = 0;
if (lump->Flags & LUMPF_SHORTNAME)
auto lflags = file->GetEntryFlags(fileindex);
if (!name) name = file->getName(fileindex);
if (lflags & RESFF_SHORTNAME)
{
UpperCopy(shortName.String, lump->getName());
UpperCopy(shortName.String, name);
shortName.String[8] = 0;
LongName = "";
Namespace = lump->GetNamespace();
Namespace = file->GetEntryNamespace(fileindex);
resourceId = -1;
}
else if ((lump->Flags & LUMPF_EMBEDDED) || !lump->getName() || !*lump->getName())
else if ((lflags & RESFF_EMBEDDED) || !name || !*name)
{
shortName.qword = 0;
LongName = "";
@ -121,8 +129,8 @@ struct FileSystem::LumpRecord
}
else
{
LongName = lump->getName();
resourceId = lump->GetIndexNum();
LongName = name;
resourceId = file->GetEntryResourceID(fileindex);
// Map some directories to WAD namespaces.
// Note that some of these namespaces don't exist in WADS.
@ -206,11 +214,6 @@ void FileSystem::DeleteAll ()
Hashes.clear();
NumEntries = 0;
// explicitly delete all manually added lumps.
for (auto &frec : FileInfo)
{
if (frec.rfnum == -1) delete frec.lump;
}
FileInfo.clear();
for (int i = (int)Files.size() - 1; i >= 0; --i)
{
@ -242,6 +245,8 @@ bool FileSystem::InitMultipleFiles (std::vector<std::string>& filenames, LumpFil
{
int numfiles;
// the first call here will designate a main thread which may use shared file readers. All other thewads have to open new file handles.
SetMainThread();
// open all the files, load headers, and count lumps
DeleteAll();
numfiles = 0;
@ -295,23 +300,23 @@ bool FileSystem::InitMultipleFiles (std::vector<std::string>& filenames, LumpFil
//
//==========================================================================
FResourceFile* CheckLump(const char* filename, FileReader& file, LumpFilterInfo*, FileSystemMessageFunc Printf, StringPool* sp);
int FileSystem::AddFromBuffer(const char* name, char* data, int size, int id, int flags)
{
FileReader fr;
fr.OpenMemoryArray((uint8_t*)data, size);
FileData blob(data, size);
fr.OpenMemoryArray(blob);
// just wrap this into a single lump resource file (should be done a little better later.)
auto rf = CheckLump(name, fr, nullptr, nullptr, stringpool);
if (rf)
{
Files.push_back(rf);
FResourceLump* lump = rf->GetLump(0);
FileInfo.resize(FileInfo.size() + 1);
FileSystem::LumpRecord* lump_p = &FileInfo.back();
lump_p->SetFromLump((int)Files.size(), lump, stringpool);
}
// wrap this into a single lump resource file (should be done a little better later.)
auto rf = new FResourceFile(name, fr, stringpool);
auto Entries = rf->AllocateEntries(1);
Entries[0].FileName = rf->NormalizeFileName(ExtractBaseName(name, true).c_str());
Entries[0].ResourceID = -1;
Entries[0].Length = size;
Files.push_back(rf);
FileInfo.resize(FileInfo.size() + 1);
FileSystem::LumpRecord* lump_p = &FileInfo.back();
lump_p->SetFromLump(rf, 0, (int)Files.size() - 1, stringpool);
return (int)FileInfo.size() - 1;
}
@ -377,25 +382,23 @@ void FileSystem::AddFile (const char *filename, FileReader *filer, LumpFilterInf
uint32_t lumpstart = (uint32_t)FileInfo.size();
resfile->SetFirstLump(lumpstart);
Files.push_back(resfile);
for (int i = 0; i < resfile->EntryCount(); i++)
{
FResourceLump* lump = resfile->GetLump(i);
FileInfo.resize(FileInfo.size() + 1);
FileSystem::LumpRecord* lump_p = &FileInfo.back();
lump_p->SetFromLump((int)Files.size(), lump, stringpool);
lump_p->SetFromLump(resfile, i, (int)Files.size() - 1, stringpool);
}
Files.push_back(resfile);
for (int i = 0; i < resfile->EntryCount(); i++)
{
int flags = resfile->GetEntryFlags(i);
if (flags & LUMPF_EMBEDDED)
if (flags & RESFF_EMBEDDED)
{
std::string path = filename;
path += ':';
path += resfile->getName(i);
auto embedded = resfile->GetEntryReader(i, true);
auto embedded = resfile->GetEntryReader(i, READER_NEW, READERFLAG_SEEKABLE);
AddFile(path.c_str(), &embedded, filter, Printf, hashfile);
}
}
@ -425,9 +428,9 @@ void FileSystem::AddFile (const char *filename, FileReader *filer, LumpFilterInf
for (int i = 0; i < resfile->EntryCount(); i++)
{
int flags = resfile->GetEntryFlags(i);
if (!(flags & LUMPF_EMBEDDED))
if (!(flags & RESFF_EMBEDDED))
{
auto reader = resfile->GetEntryReader(i, true);
auto reader = resfile->GetEntryReader(i, READER_SHARED, 0);
md5Hash(filereader, cksum);
for (size_t j = 0; j < sizeof(cksum); ++j)
@ -529,8 +532,9 @@ int FileSystem::CheckNumForName (const char *name, int space) const
// If we find a lump with this name in the global namespace that does not come
// from a Zip return that. WADs don't know these namespaces and single lumps must
// work as well.
auto lflags = lump.resfile->GetEntryFlags(lump.resindex);
if (space > ns_specialzipdirectory && lump.Namespace == ns_global &&
!((lump.lump->Flags ^lump.flags) & LUMPF_FULLPATH)) break;
!((lflags ^lump.flags) & RESFF_FULLPATH)) break;
}
i = NextLumpIndex[i];
}
@ -770,31 +774,14 @@ int FileSystem::GetResource (int resid, const char *type, int filenum) const
//
//==========================================================================
int FileSystem::FileLength (int lump) const
ptrdiff_t FileSystem::FileLength (int lump) const
{
if ((size_t)lump >= NumEntries)
{
return -1;
}
return FileInfo[lump].lump->LumpSize;
}
//==========================================================================
//
// GetFileOffset
//
// Returns the offset from the beginning of the file to the lump.
// Returns -1 if the lump is compressed or can't be read directly
//
//==========================================================================
int FileSystem::GetFileOffset (int lump)
{
if ((size_t)lump >= NumEntries)
{
return -1;
}
return FileInfo[lump].lump->GetFileOffset();
const auto &lump_p = FileInfo[lump];
return (int)lump_p.resfile->Length(lump_p.resindex);
}
//==========================================================================
@ -810,7 +797,8 @@ int FileSystem::GetFileFlags (int lump)
return 0;
}
return FileInfo[lump].lump->Flags ^ FileInfo[lump].flags;
const auto& lump_p = FileInfo[lump];
return lump_p.resfile->GetEntryFlags(lump_p.resindex) ^ lump_p.flags;
}
//==========================================================================
@ -906,8 +894,6 @@ void FileSystem::RenameFile(int num, const char* newfn)
//
//==========================================================================
static FResourceLump placeholderLump;
void FileSystem::MoveLumpsInFolder(const char *path)
{
if (FileInfo.size() == 0)
@ -925,11 +911,12 @@ void FileSystem::MoveLumpsInFolder(const char *path)
if (li.rfnum >= GetIwadNum()) break;
if (strnicmp(li.LongName, path, len) == 0)
{
FileInfo.push_back(li);
li.lump = &placeholderLump; // Make the old entry point to something empty. We cannot delete the lump record here because it'd require adjustment of all indices in the list.
auto lic = li; // make a copy before pushing.
FileInfo.push_back(lic);
li.LongName = ""; //nuke the name of the old record.
li.shortName.qword = 0;
auto &ln = FileInfo.back();
ln.lump->LumpNameSetup(ln.LongName + len, stringpool); // may be able to avoid the string allocation!
ln.SetFromLump(rfnum, ln.lump, stringpool);
ln.SetFromLump(li.resfile, li.resindex, rfnum, stringpool, ln.LongName + len);
}
}
}
@ -1309,9 +1296,7 @@ FileData FileSystem::ReadFile (int lump)
{
throw FileSystemException("ReadFile: %u >= NumEntries", lump);
}
auto lumpp = FileInfo[lump].lump;
return FileData(lumpp);
return FileInfo[lump].resfile->Read(FileInfo[lump].resindex);
}
//==========================================================================
@ -1323,53 +1308,31 @@ FileData FileSystem::ReadFile (int lump)
//==========================================================================
FileReader FileSystem::OpenFileReader(int lump)
FileReader FileSystem::OpenFileReader(int lump, int readertype, int readerflags)
{
if ((unsigned)lump >= (unsigned)FileInfo.size())
{
throw FileSystemException("OpenFileReader: %u >= NumEntries", lump);
}
auto rl = FileInfo[lump].lump;
auto rd = rl->GetReader();
if (rl->RefCount == 0 && rd != nullptr && !rd->GetBuffer() && !(rl->Flags & LUMPF_COMPRESSED))
{
FileReader rdr;
rdr.OpenFilePart(*rd, rl->GetFileOffset(), rl->LumpSize);
return rdr;
}
return rl->NewReader(); // This always gets a reader to the cache
}
FileReader FileSystem::ReopenFileReader(int lump, bool alwayscache)
{
if ((unsigned)lump >= (unsigned)FileInfo.size())
{
throw FileSystemException("ReopenFileReader: %u >= NumEntries", lump);
}
auto rl = FileInfo[lump].lump;
auto rd = rl->GetReader();
if (rl->RefCount == 0 && rd != nullptr && !rd->GetBuffer() && !alwayscache && !(rl->Flags & LUMPF_COMPRESSED))
{
int fileno = GetFileContainer(lump);
const char *filename = GetResourceFileFullName(fileno);
FileReader fr;
if (fr.OpenFile(filename, rl->GetFileOffset(), rl->LumpSize))
{
return fr;
}
}
return rl->NewReader(); // This always gets a reader to the cache
auto file = FileInfo[lump].resfile;
return file->GetEntryReader(FileInfo[lump].resindex, readertype, readerflags);
}
FileReader FileSystem::OpenFileReader(const char* name)
{
FileReader fr;
auto lump = CheckNumForFullName(name);
if (lump < 0) return FileReader();
else return OpenFileReader(lump);
if (lump >= 0) fr = OpenFileReader(lump);
return fr;
}
FileReader FileSystem::ReopenFileReader(const char* name, bool alwayscache)
{
FileReader fr;
auto lump = CheckNumForFullName(name);
if (lump >= 0) fr = ReopenFileReader(lump, alwayscache);
return fr;
}
//==========================================================================
@ -1498,7 +1461,7 @@ bool FileSystem::CreatePathlessCopy(const char *name, int id, int /*flags*/)
if (slash == nullptr)
{
FileInfo[lump].flags = LUMPF_FULLPATH;
FileInfo[lump].flags = RESFF_FULLPATH;
return true; // already is pathless.
}
@ -1506,7 +1469,7 @@ bool FileSystem::CreatePathlessCopy(const char *name, int id, int /*flags*/)
// just create a new reference to the original data with a different name.
oldlump.LongName = slash + 1;
oldlump.resourceId = id;
oldlump.flags = LUMPF_FULLPATH;
oldlump.flags = RESFF_FULLPATH;
FileInfo.push_back(oldlump);
return true;
}

5
source/common/filesystem/source/fs_stringpool.cpp
View file

@ -104,10 +104,11 @@ StringPool::Block *StringPool::AddBlock(size_t size)
return mem;
}
void *StringPool::iAlloc(size_t size)
void *StringPool::Alloc(size_t size)
{
Block *block;
size = (size + 7) & ~7;
for (block = TopBlock; block != nullptr; block = block->NextBlock)
{
void *res = block->Alloc(size);
@ -122,7 +123,7 @@ void *StringPool::iAlloc(size_t size)
const char* StringPool::Strdup(const char* str)
{
char* p = (char*)iAlloc((strlen(str) + 8) & ~7 );
char* p = (char*)Alloc(strlen(str) + 1);
strcpy(p, str);
return p;
}

2
source/common/filesystem/source/fs_stringpool.h
View file

@ -12,12 +12,12 @@ private:
public:
~StringPool();
const char* Strdup(const char*);
void* Alloc(size_t size);
protected:
struct Block;
Block *AddBlock(size_t size);
void *iAlloc(size_t size);
Block *TopBlock;
Block *FreeBlocks;

643
source/common/filesystem/source/resourcefile.cpp
View file

@ -35,13 +35,29 @@
*/
#include <miniz.h>
#include "resourcefile_internal.h"
#include "resourcefile.h"
#include "md5.hpp"
#include "fs_stringpool.h"
#include "files_internal.h"
#include "unicode.h"
#include "fs_findfile.h"
#include "fs_decompress.h"
#include "wildcards.hpp"
namespace FileSys {
// this is for restricting shared file readers to the main thread.
thread_local bool mainThread;
void SetMainThread()
{
// only set the global flag on the first thread calling this.
static bool done = false;
if (!done)
{
mainThread = done = true;
}
}
std::string ExtractBaseName(const char* path, bool include_extension)
{
const char* src, * dot;
@ -78,195 +94,43 @@ void strReplace(std::string& str, const char *from, const char* to)
}
}
//==========================================================================
//
// File reader that reads from a lump's cache
//
//==========================================================================
class FLumpReader : public MemoryReader
{
FResourceLump *source;
public:
FLumpReader(FResourceLump *src)
: MemoryReader(NULL, src->LumpSize), source(src)
{
src->Lock();
bufptr = src->Cache;
}
~FLumpReader()
{
source->Unlock();
}
};
//==========================================================================
//
// Base class for resource lumps
//
//==========================================================================
FResourceLump::~FResourceLump()
{
if (Cache != NULL && RefCount >= 0)
{
delete [] Cache;
Cache = NULL;
}
Owner = NULL;
}
//==========================================================================
//
// Sets up the lump name information for anything not coming from a WAD file.
//
//==========================================================================
void FResourceLump::LumpNameSetup(const char *iname, StringPool* allocator)
{
// this causes interference with real Dehacked lumps.
if (!stricmp(iname, "dehacked.exe"))
{
iname = "";
}
else if (allocator)
{
iname = allocator->Strdup(iname);
}
FullName = iname;
}
//==========================================================================
//
// Checks for embedded resource files
//
//==========================================================================
static bool IsWadInFolder(const FResourceFile* const archive, const char* const resPath)
bool FResourceFile::IsFileInFolder(const char* const resPath)
{
// Checks a special case when <somefile.wad> was put in
// <myproject> directory inside <myproject.zip>
if (NULL == archive)
{
return false;
}
const auto dirName = ExtractBaseName(archive->GetFileName());
const auto dirName = ExtractBaseName(FileName);
const auto fileName = ExtractBaseName(resPath, true);
const std::string filePath = dirName + '/' + fileName;
return 0 == stricmp(filePath.c_str(), resPath);
}
void FResourceLump::CheckEmbedded(LumpFilterInfo* lfi)
void FResourceFile::CheckEmbedded(uint32_t entry, LumpFilterInfo* lfi)
{
// Checks for embedded archives
const char *c = strstr(FullName, ".wad");
if (c && strlen(c) == 4 && (!strchr(FullName, '/') || IsWadInFolder(Owner, FullName)))
auto FullName = Entries[entry].FileName;
const char *c = strstr(FullName, ".wad"); // fixme: Use lfi for this.
if (c && strlen(c) == 4 && (!strchr(FullName, '/') || IsFileInFolder(FullName)))
{
Flags |= LUMPF_EMBEDDED;
Entries[entry].Flags |= RESFF_EMBEDDED;
}
else if (lfi) for (auto& fstr : lfi->embeddings)
{
if (!stricmp(FullName, fstr.c_str()))
{
Flags |= LUMPF_EMBEDDED;
Entries[entry].Flags |= RESFF_EMBEDDED;
}
}
}
//==========================================================================
//
// this is just for completeness. For non-Zips only an uncompressed lump can
// be returned.
//
//==========================================================================
FCompressedBuffer FResourceLump::GetRawData()
{
FCompressedBuffer cbuf = { (unsigned)LumpSize, (unsigned)LumpSize, METHOD_STORED, 0, 0, new char[LumpSize] };
memcpy(cbuf.mBuffer, Lock(), LumpSize);
Unlock();
cbuf.mCRC32 = crc32(0, (uint8_t*)cbuf.mBuffer, LumpSize);
return cbuf;
}
//==========================================================================
//
// Returns the owner's FileReader if it can be used to access this lump
//
//==========================================================================
FileReader *FResourceLump::GetReader()
{
return NULL;
}
//==========================================================================
//
// Returns a file reader to the lump's cache
//
//==========================================================================
FileReader FResourceLump::NewReader()
{
return FileReader(new FLumpReader(this));
}
//==========================================================================
//
// Caches a lump's content and increases the reference counter
//
//==========================================================================
void *FResourceLump::Lock()
{
if (Cache != NULL)
{
if (RefCount > 0) RefCount++;
}
else if (LumpSize > 0)
{
try
{
FillCache();
}
catch (const FileSystemException& err)
{
// enrich the message with info about this lump.
throw FileSystemException("%s, file '%s': %s", getName(), Owner->GetFileName(), err.what());
}
}
return Cache;
}
//==========================================================================
//
// Decrements reference counter and frees lump if counter reaches 0
//
//==========================================================================
int FResourceLump::Unlock()
{
if (LumpSize > 0 && RefCount > 0)
{
if (--RefCount == 0)
{
delete [] Cache;
Cache = NULL;
}
}
return RefCount;
}
//==========================================================================
//
// Opens a resource file
@ -282,11 +146,13 @@ FResourceFile *CheckPak(const char *filename, FileReader &file, LumpFilterInfo*
FResourceFile *CheckZip(const char *filename, FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
FResourceFile *Check7Z(const char *filename, FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
FResourceFile* CheckSSI(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
FResourceFile* CheckHog(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
FResourceFile* CheckMvl(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
FResourceFile* CheckWHRes(const char* filename, FileReader& file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
FResourceFile *CheckLump(const char *filename,FileReader &file, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
FResourceFile *CheckDir(const char *filename, bool nosub, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
static CheckFunc funcs[] = { CheckWad, CheckZip, Check7Z, CheckPak, CheckGRP, CheckRFF, CheckSSI, CheckWHRes, CheckLump };
static CheckFunc funcs[] = { CheckWad, CheckZip, Check7Z, CheckPak, CheckGRP, CheckRFF, CheckSSI, CheckHog, CheckMvl, CheckWHRes, CheckLump };
static int nulPrintf(FSMessageLevel msg, const char* fmt, ...)
{
@ -347,11 +213,95 @@ FResourceFile::~FResourceFile()
if (!stringpool->shared) delete stringpool;
}
int lumpcmp(const void * a, const void * b)
//==========================================================================
//
// this is just for completeness. For non-Zips only an uncompressed lump can
// be returned.
//
//==========================================================================
FCompressedBuffer FResourceFile::GetRawData(uint32_t entry)
{
FResourceLump * rec1 = (FResourceLump *)a;
FResourceLump * rec2 = (FResourceLump *)b;
return stricmp(rec1->getName(), rec2->getName());
size_t LumpSize = entry << NumLumps ? Entries[entry].Length : 0;
FCompressedBuffer cbuf = { LumpSize, LumpSize, METHOD_STORED, 0, 0, LumpSize == 0? nullptr : new char[LumpSize] };
if (LumpSize > 0)
{
auto fr = GetEntryReader(entry, READER_SHARED, 0);
size_t read = fr.Read(cbuf.mBuffer, LumpSize);
if (read < LumpSize)
{
delete cbuf.mBuffer;
cbuf.mBuffer = nullptr;
LumpSize = cbuf.mCompressedSize = cbuf.mSize = 0;
}
}
if (LumpSize > 0)
cbuf.mCRC32 = crc32(0, (uint8_t*)cbuf.mBuffer, LumpSize);
return cbuf;
}
//==========================================================================
//
// normalize the visible file name in the system
// to lowercase canonical precomposed Unicode.
//
//==========================================================================
const char* FResourceFile::NormalizeFileName(const char* fn, int fallbackcp)
{
if (!fn || !*fn) return "";
auto norm = tolower_normalize(fn);
if (!norm)
{
if (fallbackcp == 437)
{
std::vector<char> buffer;
ibm437_to_utf8(fn, buffer);
norm = tolower_normalize(buffer.data());
}
// maybe handle other codepages
else
{
// if the filename is not valid UTF-8, nuke all bytes larger than 0x80 so that we still got something semi-usable
std::string ffn = fn;
for (auto& c : ffn)
{
if (c & 0x80) c = '@';
}
norm = tolower_normalize(&ffn.front());
}
}
FixPathSeparator(norm);
auto pooled = stringpool->Strdup(norm);
free(norm);
return pooled;
}
//==========================================================================
//
// allocate the Entries array
// this also uses the string pool to reduce maintenance
//
//==========================================================================
FResourceEntry* FResourceFile::AllocateEntries(int count)
{
NumLumps = count;
Entries = (FResourceEntry*)stringpool->Alloc(count * sizeof(FResourceEntry));
memset(Entries, 0, count * sizeof(FResourceEntry));
return Entries;
}
//---------------------------------------------------
int entrycmp(const void* a, const void* b)
{
FResourceEntry* rec1 = (FResourceEntry*)a;
FResourceEntry* rec2 = (FResourceEntry*)b;
// we are comparing lowercase UTF-8 here
return strcmp(rec1->FileName, rec2->FileName);
}
//==========================================================================
@ -400,30 +350,132 @@ void FResourceFile::GenerateHash()
//
//==========================================================================
void FResourceFile::PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterInfo *filter)
void FResourceFile::PostProcessArchive(LumpFilterInfo *filter)
{
// Entries in archives are sorted alphabetically
qsort(lumps, NumLumps, lumpsize, lumpcmp);
// only do this for archive types which contain full file names. All others are assumed to be pre-sorted.
if (NumLumps == 0 || !(Entries[0].Flags & RESFF_FULLPATH)) return;
// First eliminate all unwanted files
if (filter)
{
for (uint32_t i = 0; i < NumLumps; i++)
{
std::string name = Entries[i].FileName;
for (auto& pattern : filter->blockednames)
{
if (wildcards::match(name, pattern))
{
Entries[i].FileName = "";
continue;
}
}
}
}
// Entries in archives are sorted alphabetically.
qsort(Entries, NumLumps, sizeof(Entries[0]), entrycmp);
if (!filter) return;
FindCommonFolder(filter);
// Filter out lumps using the same names as the Autoload.* sections
// in the ini file use. We reduce the maximum lump concidered after
// in the ini file. We reduce the maximum lump concidered after
// each one so that we don't risk refiltering already filtered lumps.
uint32_t max = NumLumps;
max -= FilterLumpsByGameType(filter, lumps, lumpsize, max);
ptrdiff_t len;
ptrdiff_t lastpos = -1;
std::string file;
std::string& LumpFilter = filter->dotFilter;
while (size_t(len = LumpFilter.find_first_of('.', lastpos+1)) != LumpFilter.npos)
for (auto& LumpFilter : filter->gameTypeFilter)
{
max -= FilterLumps(std::string(LumpFilter, 0, len), lumps, lumpsize, max);
lastpos = len;
ptrdiff_t len;
ptrdiff_t lastpos = -1;
std::string file;
while (size_t(len = LumpFilter.find_first_of('.', lastpos + 1)) != LumpFilter.npos)
{
max -= FilterLumps(std::string(LumpFilter, 0, len), max);
lastpos = len;
}
max -= FilterLumps(LumpFilter, max);
}
max -= FilterLumps(LumpFilter, lumps, lumpsize, max);
JunkLeftoverFilters(lumps, lumpsize, max);
JunkLeftoverFilters(max);
for (uint32_t i = 0; i < NumLumps; i++)
{
CheckEmbedded(i, filter);
}
}
//==========================================================================
//
// FResourceFile :: FindCommonFolder
//
// Checks if all content is in a common folder that can be stripped out.
//
//==========================================================================
void FResourceFile::FindCommonFolder(LumpFilterInfo* filter)
{
std::string name0, name1;
bool foundspeciallump = false;
bool foundprefix = false;
// try to find a path prefix.
for (uint32_t i = 0; i < NumLumps; i++)
{
if (*Entries[i].FileName == 0) continue;
std::string name = Entries[i].FileName;
// first eliminate files we do not want to have.
// Some, like MacOS resource forks and executables are eliminated unconditionally, but the calling code can alsp pass a list of invalid content.
if (name.find("filter/") == 0)
return; // 'filter' is a reserved name of the file system. If this appears in the root we got no common folder, and 'filter' cannot be it.
if (!foundprefix)
{
// check for special names, if one of these gets found this must be treated as a normal zip.
bool isspecial = name.find("/") == std::string::npos ||
(filter && std::find(filter->reservedFolders.begin(), filter->reservedFolders.end(), name) != filter->reservedFolders.end());
if (isspecial) break;
name0 = std::string(name, 0, name.rfind("/") + 1);
name1 = std::string(name, 0, name.find("/") + 1);
foundprefix = true;
}
if (name.find(name0) != 0)
{
if (!name1.empty())
{
name0 = name1;
if (name.find(name0) != 0)
{
name0 = "";
}
}
if (name0.empty())
break;
}
if (!foundspeciallump && filter)
{
// at least one of the more common definition lumps must be present.
for (auto& p : filter->requiredPrefixes)
{
if (name.find(name0 + p) == 0 || name.rfind(p) == size_t(name.length() - p.length()))
{
foundspeciallump = true;
break;
}
}
}
}
// If it ran through the list without finding anything it should not attempt any path remapping.
if (!foundspeciallump || name0.empty()) return;
size_t pathlen = name0.length();
for (uint32_t i = 0; i < NumLumps; i++)
{
if (Entries[i].FileName[0] == 0) continue;
Entries[i].FileName += pathlen;
}
}
//==========================================================================
@ -436,7 +488,7 @@ void FResourceFile::PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterI
//
//==========================================================================
int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_t lumpsize, uint32_t max)
int FResourceFile::FilterLumps(const std::string& filtername, uint32_t max)
{
uint32_t start, end;
@ -446,44 +498,35 @@ int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_
}
std::string filter = "filter/" + filtername + '/';
bool found = FindPrefixRange(filter.c_str(), lumps, lumpsize, max, start, end);
// Workaround for old Doom filter names (todo: move out of here.)
if (!found && filtername.find("doom.id.doom") == 0)
{
strReplace(filter, "doom.id.doom", "doom.doom");
found = FindPrefixRange(filter.c_str(), lumps, lumpsize, max, start, end);
}
bool found = FindPrefixRange(filter.c_str(), max, start, end);
if (found)
{
void *from = (uint8_t *)lumps + start * lumpsize;
// Remove filter prefix from every name
void *lump_p = from;
for (uint32_t i = start; i < end; ++i, lump_p = (uint8_t *)lump_p + lumpsize)
for (uint32_t i = start; i < end; ++i)
{
FResourceLump *lump = (FResourceLump *)lump_p;
assert(strnicmp(lump->FullName, filter.c_str(), filter.length()) == 0);
lump->LumpNameSetup(lump->FullName + filter.length(), nullptr);
assert(strnicmp(Entries[i].FileName, filter.c_str(), filter.length()) == 0);
Entries[i].FileName += filter.length();
}
// Move filtered lumps to the end of the lump list.
size_t count = (end - start) * lumpsize;
void *to = (uint8_t *)lumps + NumLumps * lumpsize - count;
size_t count = (end - start);
auto from = Entries + start;
auto to = Entries + NumLumps - count;
assert (to >= from);
if (from != to)
{
// Copy filtered lumps to a temporary buffer.
uint8_t *filteredlumps = new uint8_t[count];
memcpy(filteredlumps, from, count);
auto filteredlumps = new FResourceEntry[count];
memcpy(filteredlumps, from, count * sizeof(*Entries));
// Shift lumps left to make room for the filtered ones at the end.
memmove(from, (uint8_t *)from + count, (NumLumps - end) * lumpsize);
memmove(from, from + count, (NumLumps - end) * sizeof(*Entries));
// Copy temporary buffer to newly freed space.
memcpy(to, filteredlumps, count);
memcpy(to, filteredlumps, count * sizeof(*Entries));
delete[] filteredlumps;
}
@ -491,29 +534,6 @@ int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_
return end - start;
}
//==========================================================================
//
// FResourceFile :: FilterLumpsByGameType
//
// Matches any lumps that match "filter/game-<gametype>/*". Includes
// inclusive gametypes like Raven.
//
//==========================================================================
int FResourceFile::FilterLumpsByGameType(LumpFilterInfo *filter, void *lumps, size_t lumpsize, uint32_t max)
{
if (filter == nullptr)
{
return 0;
}
int count = 0;
for (auto &fstring : filter->gameTypeFilter)
{
count += FilterLumps(fstring, lumps, lumpsize, max);
}
return count;
}
//==========================================================================
//
// FResourceFile :: JunkLeftoverFilters
@ -522,19 +542,17 @@ int FResourceFile::FilterLumpsByGameType(LumpFilterInfo *filter, void *lumps, si
//
//==========================================================================
void FResourceFile::JunkLeftoverFilters(void *lumps, size_t lumpsize, uint32_t max)
void FResourceFile::JunkLeftoverFilters(uint32_t max)
{
uint32_t start, end;
if (FindPrefixRange("filter/", lumps, lumpsize, max, start, end))
if (FindPrefixRange("filter/", max, start, end))
{
// Since the resource lumps may contain non-POD data besides the
// full name, we "delete" them by erasing their names so they
// can't be found.
void *stop = (uint8_t *)lumps + end * lumpsize;
for (void *p = (uint8_t *)lumps + start * lumpsize; p < stop; p = (uint8_t *)p + lumpsize)
for (uint32_t i = start; i < end; i++)
{
FResourceLump *lump = (FResourceLump *)p;
lump->clearName();
Entries[i].FileName = "";
}
}
}
@ -549,25 +567,24 @@ void FResourceFile::JunkLeftoverFilters(void *lumps, size_t lumpsize, uint32_t m
//
//==========================================================================
bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lumpsize, uint32_t maxlump, uint32_t &start, uint32_t &end)
bool FResourceFile::FindPrefixRange(const char* filter, uint32_t maxlump, uint32_t &start, uint32_t &end)
{
uint32_t min, max, mid, inside;
FResourceLump *lump;
int cmp = 0;
end = start = 0;
// Pretend that our range starts at 1 instead of 0 so that we can avoid
// unsigned overflow if the range starts at the first lump.
lumps = (uint8_t *)lumps - lumpsize;
auto lumps = &Entries[-1];
// Binary search to find any match at all.
mid = min = 1, max = maxlump;
while (min <= max)
{
mid = min + (max - min) / 2;
lump = (FResourceLump *)((uint8_t *)lumps + mid * lumpsize);
cmp = strnicmp(lump->FullName, filter, (int)strlen(filter));
auto lump = &lumps[mid];
cmp = strnicmp(lump->FileName, filter, strlen(filter));
if (cmp == 0)
break;
else if (cmp < 0)
@ -586,8 +603,8 @@ bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lump
while (min <= max)
{
mid = min + (max - min) / 2;
lump = (FResourceLump *)((uint8_t *)lumps + mid * lumpsize);
cmp = strnicmp(lump->FullName, filter, (int)strlen(filter));
auto lump = &lumps[mid];
cmp = strnicmp(lump->FileName, filter, strlen(filter));
// Go left on matches and right on misses.
if (cmp == 0)
max = mid - 1;
@ -601,8 +618,8 @@ bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lump
while (min <= max)
{
mid = min + (max - min) / 2;
lump = (FResourceLump *)((uint8_t *)lumps + mid * lumpsize);
cmp = strnicmp(lump->FullName, filter, (int)strlen(filter));
auto lump = &lumps[mid];
cmp = strnicmp(lump->FileName, filter, strlen(filter));
// Go right on matches and left on misses.
if (cmp == 0)
min = mid + 1;
@ -621,27 +638,19 @@ bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lump
int FResourceFile::FindEntry(const char *name)
{
auto norm_fn = tolower_normalize(name);
for (unsigned i = 0; i < NumLumps; i++)
{
if (!stricmp(name, getName(i)))
if (!strcmp(norm_fn, getName(i)))
{
free(norm_fn);
return i;
}
}
free(norm_fn);
return -1;
}
//==========================================================================
//
// Caches a lump's content and increases the reference counter
//
//==========================================================================
FileReader *FUncompressedLump::GetReader()
{
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
return Owner->GetContainerReader();
}
//==========================================================================
//
@ -649,103 +658,73 @@ FileReader *FUncompressedLump::GetReader()
//
//==========================================================================
int FUncompressedLump::FillCache()
FileReader FResourceFile::GetEntryReader(uint32_t entry, int readertype, int readerflags)
{
const char * buffer = Owner->GetContainerReader()->GetBuffer();
if (buffer != NULL)
FileReader fr;
if (entry < NumLumps)
{
// This is an in-memory file so the cache can point directly to the file's data.
Cache = const_cast<char*>(buffer) + Position;
RefCount = -1;
return -1;
}
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
Cache = new char[LumpSize];
auto read = Owner->GetContainerReader()->Read(Cache, LumpSize);
if (read != LumpSize)
{
throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize);
}
RefCount = 1;
return 1;
}
//==========================================================================
//
// Base class for uncompressed resource files
//
//==========================================================================
FUncompressedFile::FUncompressedFile(const char *filename, StringPool* sp)
: FResourceFile(filename, sp)
{}
FUncompressedFile::FUncompressedFile(const char *filename, FileReader &r, StringPool* sp)
: FResourceFile(filename, r, sp)
{}
//==========================================================================
//
// external lump
//
//==========================================================================
FExternalLump::FExternalLump(const char *_filename, int filesize, StringPool* stringpool)
{
FileName = stringpool->Strdup(_filename);
if (filesize == -1)
{
FileReader f;
if (f.OpenFile(_filename))
if (Entries[entry].Flags & RESFF_NEEDFILESTART)
{
LumpSize = (int)f.GetLength();
SetEntryAddress(entry);
}
if (!(Entries[entry].Flags & RESFF_COMPRESSED))
{
auto buf = Reader.GetBuffer();
// if this is backed by a memory buffer, create a new reader directly referencing it.
if (buf != nullptr)
{
fr.OpenMemory(buf + Entries[entry].Position, Entries[entry].Length);
}
else
{
if (readertype == READER_SHARED && !mainThread)
readertype = READER_NEW;
if (readertype == READER_SHARED)
{
fr.OpenFilePart(Reader, Entries[entry].Position, Entries[entry].Length);
}
else if (readertype == READER_NEW)
{
fr.OpenFile(FileName, Entries[entry].Position, Entries[entry].Length);
}
else if (readertype == READER_CACHED)
{
Reader.Seek(Entries[entry].Position, FileReader::SeekSet);
auto data = Reader.Read(Entries[entry].Length);
fr.OpenMemoryArray(data);
}
}
}
else
{
LumpSize = 0;
FileReader fri;
if (readertype == READER_NEW || !mainThread) fri.OpenFile(FileName, Entries[entry].Position, Entries[entry].CompressedSize);
else fri.OpenFilePart(Reader, Entries[entry].Position, Entries[entry].CompressedSize);
int flags = DCF_TRANSFEROWNER | DCF_EXCEPTIONS;
if (readertype == READER_CACHED) flags |= DCF_CACHED;
else if (readerflags & READERFLAG_SEEKABLE) flags |= DCF_SEEKABLE;
OpenDecompressor(fr, fri, Entries[entry].Length, Entries[entry].Method, flags);
}
}
else
{
LumpSize = filesize;
}
return fr;
}
//==========================================================================
//
// Caches a lump's content and increases the reference counter
// For external lumps this reopens the file each time it is accessed
//
//==========================================================================
int FExternalLump::FillCache()
FileData FResourceFile::Read(int entry)
{
Cache = new char[LumpSize];
FileReader f;
if (f.OpenFile(FileName))
if (!(Entries[entry].Flags & RESFF_COMPRESSED))
{
auto read = f.Read(Cache, LumpSize);
if (read != LumpSize)
auto buf = Reader.GetBuffer();
// if this is backed by a memory buffer, we can just return a reference to the backing store.
if (buf != nullptr)
{
throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize);
return FileData(buf + Entries[entry].Position, Entries[entry].Length, false);
}
}
else
{
throw FileSystemException("unable to open file");
}
RefCount = 1;
return 1;
auto fr = GetEntryReader(entry, READER_SHARED, 0);
return fr.Read(entry < NumLumps ? Entries[entry].Length : 0);
}
}

38
source/common/filesystem/source/resourcefile_internal.h
View file

@ -1,38 +0,0 @@
#pragma once
#include "resourcefile.h"
namespace FileSys {
struct FUncompressedLump : public FResourceLump
{
int Position;
virtual FileReader *GetReader();
virtual int FillCache() override;
virtual int GetFileOffset() { return Position; }
};
// Base class for uncompressed resource files (WAD, GRP, PAK and single lumps)
class FUncompressedFile : public FResourceFile
{
protected:
TArray<FUncompressedLump> Lumps;
FUncompressedFile(const char *filename, StringPool* sp);
FUncompressedFile(const char *filename, FileReader &r, StringPool* sp);
virtual FResourceLump *GetLump(int no) { return ((unsigned)no < NumLumps)? &Lumps[no] : NULL; }
};
// should only be used internally.
struct FExternalLump : public FResourceLump
{
const char* FileName;
FExternalLump(const char *_filename, int filesize, StringPool* sp);
virtual int FillCache() override;
};
}

20
source/common/filesystem/source/unicode.cpp
View file

@ -121,7 +121,7 @@ void ibm437_to_utf8(const char* in, std::vector<char>& buffer)
while (int char1 = (uint8_t)*in++)
{
if (char1 >= 0x80) char1 = ibm437map[char1];
if (char1 >= 0x80) char1 = ibm437map[char1 - 0x80];
utf8_encode(char1, buffer);
}
buffer.push_back(0);
@ -140,4 +140,22 @@ char *tolower_normalize(const char *str)
return (char*)retval;
}
//==========================================================================
//
// validates the string for proper UTF-8
//
//==========================================================================
bool unicode_validate(const char* str)
{
while (*str != 0)
{
int cp;
auto result = utf8proc_iterate((const uint8_t*)str, -1, &cp);
if (result < 0) return false;
}
return true;
}
}

2
source/common/filesystem/source/unicode.h
View file

@ -6,7 +6,7 @@ namespace FileSys {
void utf16_to_utf8(const unsigned short* in, std::vector<char>& buffer);
void ibm437_to_utf8(const char* in, std::vector<char>& buffer);
int unicode_tolower(int c);
char *tolower_normalize(const char *str);
bool unicode_validate(const char* str);
}

1833
source/common/filesystem/source/wildcards.hpp Normal file
View file

File diff suppressed because it is too large Load diff

4
source/common/fonts/singlelumpfont.cpp
View file

@ -125,7 +125,7 @@ FSingleLumpFont::FSingleLumpFont (const char *name, int lump) : FFont(lump)
FontName = name;
auto data1 = fileSystem.ReadFile (lump);
auto data = data1.GetBytes();
auto data = data1.bytes();
if (data[0] == 0xE1 && data[1] == 0xE6 && data[2] == 0xD5 && data[3] == 0x1A)
{
@ -475,7 +475,7 @@ void FSingleLumpFont::LoadBMF(int lump, const uint8_t *data)
void FSingleLumpFont::CheckFON1Chars()
{
auto memLump = fileSystem.ReadFile(Lump);
auto data = memLump.GetBytes();
auto data = memLump.bytes();
const uint8_t* data_p;
data_p = data + 8;

2
source/common/menu/savegamemanager.cpp
View file

@ -316,7 +316,7 @@ unsigned FSavegameManagerBase::ExtractSaveData(int index)
auto pic = resf->FindEntry("savepic.png");
if (pic >= 0)
{
FileReader picreader = resf->GetEntryReader(pic);
FileReader picreader = resf->GetEntryReader(pic, FileSys::READER_NEW, FileSys::READERFLAG_SEEKABLE);
PNGHandle *png = M_VerifyPNG(picreader);
if (png != nullptr)
{

2
source/common/models/model.cpp
View file

@ -165,7 +165,7 @@ unsigned FindModel(const char * path, const char * modelfile, bool silent)
int len = fileSystem.FileLength(lump);
auto lumpd = fileSystem.ReadFile(lump);
const char * buffer = lumpd.GetString();
const char * buffer = lumpd.string();
if ( (size_t)fullname.LastIndexOf("_d.3d") == fullname.Len()-5 )
{

2
source/common/models/models_iqm.cpp
View file

@ -274,7 +274,7 @@ void IQMModel::LoadGeometry()
try
{
auto lumpdata = fileSystem.ReadFile(mLumpNum);
IQMFileReader reader(lumpdata.GetMem(), (int)lumpdata.GetSize());
IQMFileReader reader(lumpdata.data(), (int)lumpdata.size());
Vertices.Resize(NumVertices);
for (IQMVertexArray& vertexArray : VertexArrays)

4
source/common/models/models_md2.cpp
View file

@ -178,7 +178,7 @@ void FDMDModel::LoadGeometry()
{
static int axis[3] = { VX, VY, VZ };
auto lumpdata = fileSystem.ReadFile(mLumpNum);
auto buffer = lumpdata.GetString();
auto buffer = lumpdata.string();
texCoords = new FTexCoord[info.numTexCoords];
memcpy(texCoords, buffer + info.offsetTexCoords, info.numTexCoords * sizeof(FTexCoord));
@ -502,7 +502,7 @@ void FMD2Model::LoadGeometry()
static int axis[3] = { VX, VY, VZ };
uint8_t *md2_frames;
auto lumpdata = fileSystem.ReadFile(mLumpNum);
auto buffer = lumpdata.GetString();
auto buffer = lumpdata.string();
texCoords = new FTexCoord[info.numTexCoords];
memcpy(texCoords, (uint8_t*)buffer + info.offsetTexCoords, info.numTexCoords * sizeof(FTexCoord));

2
source/common/models/models_md3.cpp
View file

@ -190,7 +190,7 @@ bool FMD3Model::Load(const char * path, int lumpnum, const char * buffer, int le
void FMD3Model::LoadGeometry()
{
auto lumpdata = fileSystem.ReadFile(mLumpNum);
auto buffer = lumpdata.GetString();
auto buffer = lumpdata.string();
md3_header_t * hdr = (md3_header_t *)buffer;
md3_surface_t * surf = (md3_surface_t*)(buffer + LittleLong(hdr->Ofs_Surfaces));

4
source/common/models/models_ue1.cpp
View file

@ -71,9 +71,9 @@ void FUE1Model::LoadGeometry()
{
const char *buffer, *buffer2;
auto lump = fileSystem.ReadFile(mDataLump);
buffer = lump.GetString();
buffer = lump.string();
auto lump2 = fileSystem.ReadFile(mAnivLump);
buffer2 = lump2.GetString();
buffer2 = lump2.string();
// map structures
dhead = (const d3dhead*)(buffer);
dpolys = (const d3dpoly*)(buffer+sizeof(d3dhead));

4
source/common/models/voxels.cpp
View file

@ -162,8 +162,8 @@ FVoxel *R_LoadKVX(int lumpnum)
int i, j, n;
auto lump = fileSystem.ReadFile(lumpnum); // FileData adds an extra 0 byte to the end.
auto rawvoxel = lump.GetBytes();
int voxelsize = (int)(lump.GetSize());
auto rawvoxel = lump.bytes();
int voxelsize = (int)(lump.size());
// Oh, KVX, why couldn't you have a proper header? We'll just go through
// and collect each MIP level, doing lots of range checking, and if the

1
source/common/rendering/gl/gl_shaderprogram.cpp
View file

@ -89,7 +89,6 @@ void FShaderProgram::Compile(ShaderType type, const char *lumpName, const char *
{
int lump = fileSystem.CheckNumForFullName(lumpName);
if (lump == -1) I_FatalError("Unable to load '%s'", lumpName);
auto sp = fileSystem.ReadFile(lump);
FString code = GetStringFromLump(lump);
Compile(type, lumpName, code, defines, maxGlslVersion);

3
source/common/rendering/gles/gles_shader.cpp
View file

@ -383,11 +383,9 @@ bool FShader::Load(const char * name, const char * vert_prog_lump_, const char *
int vp_lump = fileSystem.CheckNumForFullName(vert_prog_lump.GetChars(), 0);
if (vp_lump == -1) I_Error("Unable to load '%s'", vert_prog_lump.GetChars());
auto vp_data = fileSystem.ReadFile(vp_lump);
int fp_lump = fileSystem.CheckNumForFullName(frag_prog_lump.GetChars(), 0);
if (fp_lump == -1) I_Error("Unable to load '%s'", frag_prog_lump.GetChars());
auto fp_data = fileSystem.ReadFile(fp_lump);
@ -422,7 +420,6 @@ bool FShader::Load(const char * name, const char * vert_prog_lump_, const char *
{
int pp_lump = fileSystem.CheckNumForFullName(proc_prog_lump.GetChars());
if (pp_lump == -1) I_Error("Unable to load '%s'", proc_prog_lump.GetChars());
auto ppf = fileSystem.ReadFile(pp_lump);
FString pp_data = GetStringFromLump(pp_lump);
if (pp_data.IndexOf("ProcessMaterial") < 0 && pp_data.IndexOf("SetupMaterial") < 0)

1
source/common/rendering/gles/gles_shaderprogram.cpp
View file

@ -89,7 +89,6 @@ void FShaderProgram::Compile(ShaderType type, const char *lumpName, const char *
{
int lump = fileSystem.CheckNumForFullName(lumpName);
if (lump == -1) I_FatalError("Unable to load '%s'", lumpName);
auto sp = fileSystem.ReadFile(lump);
FString code = GetStringFromLump(lump);
Compile(type, lumpName, code, defines, maxGlslVersion);
}

1
source/common/rendering/vulkan/shaders/vk_ppshader.cpp
View file

@ -69,7 +69,6 @@ FString VkPPShader::LoadShaderCode(const FString &lumpName, const FString &defin
{
int lump = fileSystem.CheckNumForFullName(lumpName.GetChars());
if (lump == -1) I_FatalError("Unable to load '%s'", lumpName.GetChars());
auto sp = fileSystem.ReadFile(lump);
FString code = GetStringFromLump(lump);
FString patchedCode;

1
source/common/rendering/vulkan/shaders/vk_shader.cpp
View file

@ -475,7 +475,6 @@ FString VkShaderManager::LoadPrivateShaderLump(const char *lumpname)
{
int lump = fileSystem.CheckNumForFullName(lumpname, 0);
if (lump == -1) I_Error("Unable to load '%s'", lumpname);
auto data = fileSystem.ReadFile(lump);
return GetStringFromLump(lump);
}

4
source/common/textures/formats/anmtexture.cpp
View file

@ -106,10 +106,10 @@ FAnmTexture::FAnmTexture (int lumpnum, int w, int h)
void FAnmTexture::ReadFrame(uint8_t *pixels, uint8_t *palette)
{
auto lump = fileSystem.ReadFile (SourceLump);
auto source = lump.GetBytes();
auto source = lump.bytes();
std::unique_ptr<anim_t> anim = std::make_unique<anim_t>(); // note that this struct is very large and should not go onto the stack!
if (ANIM_LoadAnim(anim.get(), source, (int)lump.GetSize()) >= 0)
if (ANIM_LoadAnim(anim.get(), source, (int)lump.size()) >= 0)
{
int numframes = ANIM_NumFrames(anim.get());
if (numframes >= 1)

2
source/common/textures/formats/automaptexture.cpp
View file

@ -93,7 +93,7 @@ PalettedPixels FAutomapTexture::CreatePalettedPixels(int conversion, int frame)
{
int x, y;
auto data = fileSystem.ReadFile (SourceLump);
auto indata = data.GetBytes();
auto indata = data.bytes();
PalettedPixels Pixels(Width * Height);

2
source/common/textures/formats/imgztexture.cpp
View file

@ -121,7 +121,7 @@ FIMGZTexture::FIMGZTexture (int lumpnum, uint16_t w, uint16_t h, int16_t l, int1
PalettedPixels FIMGZTexture::CreatePalettedPixels(int conversion, int frame)
{
auto lump = fileSystem.ReadFile (SourceLump);
auto imgz = (const ImageHeader *)lump.GetMem();
auto imgz = (const ImageHeader *)lump.data();
const uint8_t *data = (const uint8_t *)&imgz[1];
uint8_t *dest_p;

4
source/common/textures/formats/patchtexture.cpp
View file

@ -186,7 +186,7 @@ PalettedPixels FPatchTexture::CreatePalettedPixels(int conversion, int frame)
int x;
auto lump = fileSystem.ReadFile (SourceLump);
const patch_t *patch = (const patch_t *)lump.GetMem();
const patch_t *patch = (const patch_t *)lump.data();
maxcol = (const column_t *)((const uint8_t *)patch + fileSystem.FileLength (SourceLump) - 3);
@ -296,7 +296,7 @@ void FPatchTexture::DetectBadPatches ()
// It must be 256 pixels tall, and all its columns must have exactly
// one post, where each post has a supposed length of 0.
auto lump = fileSystem.ReadFile (SourceLump);
const patch_t *realpatch = (patch_t *)lump.GetMem();
const patch_t *realpatch = (patch_t *)lump.data();
const uint32_t *cofs = realpatch->columnofs;
int x, x2 = LittleShort(realpatch->width);

6
source/common/textures/formats/qoitexture.cpp
View file

@ -141,14 +141,14 @@ int FQOITexture::CopyPixels(FBitmap *bmp, int conversion, int frame)
constexpr auto QOI_COLOR_HASH = [](PalEntry C) { return (C.r * 3 + C.g * 5 + C.b * 7 + C.a * 11); };
auto lump = fileSystem.ReadFile(SourceLump);
if (lump.GetSize() < 22) return 0; // error
if (lump.size() < 22) return 0; // error
PalEntry index[64] = {};
PalEntry pe = 0xff000000;
size_t p = 14, run = 0;
size_t chunks_len = lump.GetSize() - 8;
auto bytes = lump.GetBytes();
size_t chunks_len = lump.size() - 8;
auto bytes = lump.bytes();
for (int h = 0; h < Height; h++)
{

6
source/common/textures/formats/rawpagetexture.cpp
View file

@ -183,7 +183,7 @@ FRawPageTexture::FRawPageTexture (int lumpnum)
PalettedPixels FRawPageTexture::CreatePalettedPixels(int conversion, int frame)
{
auto lump = fileSystem.ReadFile (SourceLump);
auto source = lump.GetBytes();
auto source = lump.bytes();
const uint8_t *source_p = source;
uint8_t *dest_p;
@ -216,8 +216,8 @@ int FRawPageTexture::CopyPixels(FBitmap *bmp, int conversion, int frame)
{
auto lump = fileSystem.ReadFile(SourceLump);
auto plump = fileSystem.ReadFile(mPaletteLump);
auto source = lump.GetBytes();
auto psource = plump.GetBytes();
auto source = lump.bytes();
auto psource = plump.bytes();
PalEntry paldata[256];
for (auto & pe : paldata)
{

12
source/common/textures/formats/startuptexture.cpp
View file

@ -165,7 +165,7 @@ FStartupTexture::FStartupTexture (int lumpnum)
bUseGamePalette = false;
auto lump = fileSystem.ReadFile (SourceLump);
auto source = lump.GetBytes();
auto source = lump.bytes();
// Initialize the bitmap palette.
// the palette is static so that the notches can share it.
@ -234,7 +234,7 @@ void PlanarToChunky(T* dest, const uint8_t* src, const T* remap, int width, int
PalettedPixels FStartupTexture::CreatePalettedPixels(int conversion, int frame)
{
auto lump = fileSystem.ReadFile (SourceLump);
auto source = lump.GetBytes();
auto source = lump.bytes();
const uint8_t *remap = ImageHelpers::GetRemap(conversion == luminance);
@ -254,7 +254,7 @@ PalettedPixels FStartupTexture::CreatePalettedPixels(int conversion, int frame)
int FStartupTexture::CopyPixels(FBitmap *bmp, int conversion, int frame)
{
auto lump = fileSystem.ReadFile (SourceLump);
auto source = lump.GetBytes();
auto source = lump.bytes();
PlanarToChunky((uint32_t*)bmp->GetPixels(), source + 48, startuppalette32, Width, Height);
return 0;
}
@ -282,7 +282,7 @@ FNotchTexture::FNotchTexture (int lumpnum, int width, int height)
PalettedPixels FNotchTexture::CreatePalettedPixels(int conversion, int frame)
{
auto lump = fileSystem.ReadFile (SourceLump);
auto source = lump.GetBytes();
auto source = lump.bytes();
const uint8_t *remap = ImageHelpers::GetRemap(conversion == luminance);
TArray<uint8_t> Work(Width*Height, true);
@ -305,7 +305,7 @@ PalettedPixels FNotchTexture::CreatePalettedPixels(int conversion, int frame)
int FNotchTexture::CopyPixels(FBitmap *bmp, int conversion, int frame)
{
auto lump = fileSystem.ReadFile (SourceLump);
auto source = lump.GetBytes();
auto source = lump.bytes();
auto Work = (uint32_t*)bmp->GetPixels();
for(int i = 0; i < Width * Height / 2; i++)
@ -339,7 +339,7 @@ FStrifeStartupTexture::FStrifeStartupTexture (int lumpnum, int w, int h)
PalettedPixels FStrifeStartupTexture::CreatePalettedPixels(int conversion, int frame)
{
auto lump = fileSystem.ReadFile (SourceLump);
auto source = lump.GetBytes();
auto source = lump.bytes();
PalettedPixels Pixels(Width*Height);
const uint8_t *remap = ImageHelpers::GetRemap(conversion == luminance);
ImageHelpers::FlipNonSquareBlockRemap(Pixels.Data(), source, Width, Height, Width, remap);

2
source/common/textures/formats/webptexture.cpp
View file

@ -142,7 +142,7 @@ int FWebPTexture::CopyPixels(FBitmap *bmp, int conversion, int frame)
config.output.u.RGBA.stride = bmp->GetPitch();
config.output.is_external_memory = 1;
(void)WebPDecode(bytes.GetBytes(), bytes.GetSize(), &config);
(void)WebPDecode(bytes.bytes(), bytes.size(), &config);
return 0;
}

4
source/common/textures/multipatchtexturebuilder.cpp
View file

@ -398,12 +398,12 @@ void FMultipatchTextureBuilder::AddTexturesLumps(int lump1, int lump2, int patch
if (lump1 >= 0)
{
auto texdir = fileSystem.ReadFile(lump1);
AddTexturesLump(texdir.GetMem(), fileSystem.FileLength(lump1), lump1, patcheslump, firstdup, true);
AddTexturesLump(texdir.data(), fileSystem.FileLength(lump1), lump1, patcheslump, firstdup, true);
}
if (lump2 >= 0)
{
auto texdir = fileSystem.ReadFile(lump2);
AddTexturesLump(texdir.GetMem(), fileSystem.FileLength(lump2), lump2, patcheslump, firstdup, false);
AddTexturesLump(texdir.data(), fileSystem.FileLength(lump2), lump2, patcheslump, firstdup, false);
}
}

6
source/common/textures/texturemanager.cpp
View file

@ -585,7 +585,7 @@ void FTextureManager::AddGroup(int wadnum, int ns, ETextureType usetype)
}
progressFunc();
}
else if (ns == ns_flats && fileSystem.GetFileFlags(firsttx) & LUMPF_MAYBEFLAT)
else if (ns == ns_flats && fileSystem.GetFileFlags(firsttx) & RESFF_MAYBEFLAT)
{
if (fileSystem.CheckNumForName(Name, ns) < firsttx)
{
@ -971,7 +971,7 @@ void FTextureManager::AddTexturesForWad(int wadnum, FMultipatchTextureBuilder &b
if (ns == ns_global)
{
// In Zips all graphics must be in a separate namespace.
if (fileSystem.GetFileFlags(i) & LUMPF_FULLPATH) continue;
if (fileSystem.GetFileFlags(i) & RESFF_FULLPATH) continue;
// Ignore lumps with empty names.
if (fileSystem.CheckFileName(i, "")) continue;
@ -1425,7 +1425,7 @@ int FTextureManager::GuesstimateNumTextures ()
break;
default:
if (fileSystem.GetFileFlags(i) & LUMPF_MAYBEFLAT) numtex++;
if (fileSystem.GetFileFlags(i) & RESFF_MAYBEFLAT) numtex++;
break;
}

2
source/common/utility/cmdlib.cpp
View file

@ -1045,7 +1045,7 @@ void uppercopy(char* to, const char* from)
FString GetStringFromLump(int lump, bool zerotruncate)
{
auto fd = fileSystem.ReadFile(lump);
FString ScriptBuffer(fd.GetString(), fd.GetSize());
FString ScriptBuffer(fd.string(), fd.size());
if (zerotruncate) ScriptBuffer.Truncate(strlen(ScriptBuffer.GetChars())); // this is necessary to properly truncate the generated string to not contain 0 bytes.
return ScriptBuffer;
}

12
source/common/utility/palette.cpp
View file

@ -681,8 +681,8 @@ FString V_GetColorStringByName(const char* name, FScriptPosition* sc)
}
auto rgbNames = fileSystem.ReadFile(rgblump);
rgb = rgbNames.GetString();
rgbEnd = rgb + rgbNames.GetSize();
rgb = rgbNames.string();
rgbEnd = rgb + rgbNames.size();
step = 0;
namelen = strlen(name);
@ -930,11 +930,11 @@ int ReadPalette(int lumpnum, uint8_t* buffer)
return 0;
}
auto lump = fileSystem.ReadFile(lumpnum);
auto lumpmem = lump.GetBytes();
auto lumpmem = lump.bytes();
memset(buffer, 0, 768);
FileReader fr;
fr.OpenMemory(lumpmem, lump.GetSize());
fr.OpenMemory(lumpmem, lump.size());
auto png = M_VerifyPNG(fr);
if (png)
{
@ -964,7 +964,7 @@ int ReadPalette(int lumpnum, uint8_t* buffer)
{
FScanner sc;
sc.OpenMem(fileSystem.GetFileFullName(lumpnum), (char*)lumpmem, int(lump.GetSize()));
sc.OpenMem(fileSystem.GetFileFullName(lumpnum), (char*)lumpmem, int(lump.size()));
sc.MustGetString();
sc.MustGetNumber(); // version - ignore
sc.MustGetNumber();
@ -982,7 +982,7 @@ int ReadPalette(int lumpnum, uint8_t* buffer)
}
else
{
memcpy(buffer, lumpmem, min<size_t>(768, lump.GetSize()));
memcpy(buffer, lumpmem, min<size_t>(768, lump.size()));
return 256;
}
}

239
source/common/utility/writezip.cpp Normal file
View file

@ -0,0 +1,239 @@
/*
** writezip.cpp
**
**---------------------------------------------------------------------------
** Copyright 1998-2009 Randy Heit
** Copyright 2005-2023 Christoph Oelckers
** All rights reserved.
**
** Redistribution and use in source and binary forms, with or without
** modification, are permitted provided that the following conditions
** are met:
**
** 1. Redistributions of source code must retain the above copyright
** notice, this list of conditions and the following disclaimer.
** 2. Redistributions in binary form must reproduce the above copyright
** notice, this list of conditions and the following disclaimer in the
** documentation and/or other materials provided with the distribution.
** 3. The name of the author may not be used to endorse or promote products
** derived from this software without specific prior written permission.
**
** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**---------------------------------------------------------------------------
**
**
*/
#include <stdint.h>
#include <algorithm>
#include "tarray.h"
#include "files.h"
#include "m_swap.h"
#include "w_zip.h"
using FileSys::FCompressedBuffer;
//==========================================================================
//
// time_to_dos
//
// Converts time from struct tm to the DOS format used by zip files.
//
//==========================================================================
static std::pair<uint16_t, uint16_t> time_to_dos(struct tm *time)
{
std::pair<uint16_t, uint16_t> val;
if (time == NULL || time->tm_year < 80)
{
val.first = val.second = 0;
}
else
{
val.first = (time->tm_year - 80) * 512 + (time->tm_mon + 1) * 32 + time->tm_mday;
val.second= time->tm_hour * 2048 + time->tm_min * 32 + time->tm_sec / 2;
}
return val;
}
//==========================================================================
//
// append_to_zip
//
// Write a given file to the zipFile.
//
// zipfile: zip object to be written to
//
// returns: position = success, -1 = error
//
//==========================================================================
static int AppendToZip(FileWriter *zip_file, const FCompressedBuffer &content, std::pair<uint16_t, uint16_t> &dostime)
{
FZipLocalFileHeader local;
int position;
int flags = 0;
int method = content.mMethod;
if (method >= FileSys::METHOD_IMPLODE_MIN && method <= FileSys::METHOD_IMPLODE_MAX)
{
flags = method - FileSys::METHOD_IMPLODE_MIN;
method = FileSys::METHOD_IMPLODE;
}
else if (method == FileSys::METHOD_DEFLATE)
{
flags = 2;
}
else if (method >= 1337)
return -1;
local.Magic = ZIP_LOCALFILE;
local.VersionToExtract[0] = 20;
local.VersionToExtract[1] = 0;
local.Flags = LittleShort((uint16_t)flags);
local.Method = LittleShort((uint16_t)method);
local.ModDate = LittleShort(dostime.first);
local.ModTime = LittleShort(dostime.second);
local.CRC32 = content.mCRC32;
local.UncompressedSize = LittleLong((unsigned)content.mSize);
local.CompressedSize = LittleLong((unsigned)content.mCompressedSize);
local.NameLength = LittleShort((unsigned short)strlen(content.filename));
local.ExtraLength = 0;
// Fill in local directory header.
position = (int)zip_file->Tell();
// Write out the header, file name, and file data.
if (zip_file->Write(&local, sizeof(local)) != sizeof(local) ||
zip_file->Write(content.filename, strlen(content.filename)) != strlen(content.filename) ||
zip_file->Write(content.mBuffer, content.mCompressedSize) != content.mCompressedSize)
{
return -1;
}
return position;
}
//==========================================================================
//
// write_central_dir
//
// Writes the central directory entry for a file.
//
//==========================================================================
int AppendCentralDirectory(FileWriter *zip_file, const FCompressedBuffer &content, std::pair<uint16_t, uint16_t> &dostime, int position)
{
FZipCentralDirectoryInfo dir;
int flags = 0;
int method = content.mMethod;
if (method >= FileSys::METHOD_IMPLODE_MIN && method <= FileSys::METHOD_IMPLODE_MAX)
{
flags = method - FileSys::METHOD_IMPLODE_MIN;
method = FileSys::METHOD_IMPLODE;
}
else if (method == FileSys::METHOD_DEFLATE)
{
flags = 2;
}
else if (method >= 1337)
return -1;
dir.Magic = ZIP_CENTRALFILE;
dir.VersionMadeBy[0] = 20;
dir.VersionMadeBy[1] = 0;
dir.VersionToExtract[0] = 20;
dir.VersionToExtract[1] = 0;
dir.Flags = LittleShort((uint16_t)flags);
dir.Method = LittleShort((uint16_t)method);
dir.ModTime = LittleShort(dostime.first);
dir.ModDate = LittleShort(dostime.second);
dir.CRC32 = content.mCRC32;
dir.CompressedSize32 = LittleLong((unsigned)content.mCompressedSize);
dir.UncompressedSize32 = LittleLong((unsigned)content.mSize);
dir.NameLength = LittleShort((unsigned short)strlen(content.filename));
dir.ExtraLength = 0;
dir.CommentLength = 0;
dir.StartingDiskNumber = 0;
dir.InternalAttributes = 0;
dir.ExternalAttributes = 0;
dir.LocalHeaderOffset32 = LittleLong((unsigned)position);
if (zip_file->Write(&dir, sizeof(dir)) != sizeof(dir) ||
zip_file->Write(content.filename, strlen(content.filename)) != strlen(content.filename))
{
return -1;
}
return 0;
}
bool WriteZip(const char* filename, const FCompressedBuffer* content, size_t contentcount)
{
// try to determine local time
struct tm *ltime;
time_t ttime;
ttime = time(nullptr);
ltime = localtime(&ttime);
auto dostime = time_to_dos(ltime);
TArray<int> positions;
auto f = FileWriter::Open(filename);
if (f != nullptr)
{
for (size_t i = 0; i < contentcount; i++)
{
int pos = AppendToZip(f, content[i], dostime);
if (pos == -1)
{
delete f;
remove(filename);
return false;
}
positions.Push(pos);
}
int dirofs = (int)f->Tell();
for (size_t i = 0; i < contentcount; i++)
{
if (AppendCentralDirectory(f, content[i], dostime, positions[i]) < 0)
{
delete f;
remove(filename);
return false;
}
}
// Write the directory terminator.
FZipEndOfCentralDirectory dirend;
dirend.Magic = ZIP_ENDOFDIR;
dirend.DiskNumber = 0;
dirend.FirstDisk = 0;
dirend.NumEntriesOnAllDisks = dirend.NumEntries = LittleShort((uint16_t)contentcount);
dirend.DirectoryOffset = LittleLong((unsigned)dirofs);
dirend.DirectorySize = LittleLong((uint32_t)(f->Tell() - dirofs));
dirend.ZipCommentLength = 0;
if (f->Write(&dirend, sizeof(dirend)) != sizeof(dirend))
{
delete f;
remove(filename);
return false;
}
delete f;
return true;
}
return false;
}

7
source/core/initfs.cpp
View file

@ -433,10 +433,9 @@ void InitFileSystem(TArray<GrpEntry>& groups)
lfi.embeddings = { "blood.rff", "sounds.rff" };
}
lfi.dotFilter = LumpFilter.GetChars();
if (isDukeEngine()) lfi.gameTypeFilter.push_back("DukeEngine");
if (isDukeLike()) lfi.gameTypeFilter.push_back("DukeLike");
lfi.gameTypeFilter.push_back(LumpFilter.GetChars());
lfi.postprocessFunc = [&]()
{
@ -448,8 +447,8 @@ void InitFileSystem(TArray<GrpEntry>& groups)
FILE* f = fopen("filesystem.dir", "wb");
for (int num = 0; num < fileSystem.GetNumEntries(); num++)
{
auto fd = fileSystem.FileLength(num);
fprintf(f, "%.50s %60s %d\n", fileSystem.GetFileFullName(num), fileSystem.GetResourceFileFullName(fileSystem.GetFileContainer(num)), fd);
int64_t fd = fileSystem.FileLength(num);
fprintf(f, "%.50s %60s %lld\n", fileSystem.GetFileFullName(num), fileSystem.GetResourceFileFullName(fileSystem.GetFileContainer(num)), fd);
}
fclose(f);
}

6
source/core/savegamehelp.cpp
View file

@ -69,6 +69,7 @@
#include "buildtiles.h"
#include "fs_findfile.h"
#include "resourcefile.h"
@ -79,6 +80,7 @@ extern FString BackupSaveGame;
int SaveVersion;
void SerializeMap(FSerializer &arc);
bool WriteZip(const char* filename, const FileSys::FCompressedBuffer* content, size_t contentcount);
BEGIN_BLD_NS
FSerializer& Serialize(FSerializer& arc, const char* keyname, XWALL& w, XWALL* def);
@ -132,7 +134,7 @@ static void SerializeSession(FSerializer& arc)
bool ReadSavegame(const char* name)
{
auto savereader = FResourceFile::OpenResourceFile(name, true);
auto savereader = FileSys::FResourceFile::OpenResourceFile(name, true);
if (savereader != nullptr)
{
@ -245,7 +247,7 @@ bool WriteSavegame(const char* filename, const char *name)
M_FinishPNG(&savepic);
auto picdata = savepic.GetBuffer();
FileSys::FCompressedBuffer bufpng = { picdata->size(), picdata->size(), FileSys::METHOD_STORED, 0, static_cast<unsigned int>(crc32(0, &(*picdata)[0], picdata->size())), (char*)&(*picdata)[0] };
FileSys::FCompressedBuffer bufpng = { picdata->size(), picdata->size(), FileSys::METHOD_STORED, static_cast<unsigned int>(crc32(0, &(*picdata)[0], picdata->size())), (char*)&(*picdata)[0] };
TArray<FileSys::FCompressedBuffer> savegame_content;
TArray<FString> savegame_filenames;

8
source/games/blood/src/blood.cpp
View file

@ -560,8 +560,8 @@ void GameInterface::loadPalette(void)
for (int i = 0; i < 5; i++)
{
auto pal = fileSystem.ReadFileFullName(PAL[i]);
if (pal.GetSize() < 768) I_FatalError("%s: file too small", PAL[i]);
paletteSetColorTable(i, (const uint8_t*)pal.GetMem(), false, false);
if (pal.size() < 768) I_FatalError("%s: file too small", PAL[i]);
paletteSetColorTable(i, pal.bytes(), false, false);
}
numshades = 64;
@ -574,12 +574,12 @@ void GameInterface::loadPalette(void)
else continue;
}
auto data = fileSystem.ReadFile(lump);
if (data.GetSize() != 64 * 256)
if (data.size() != 64 * 256)
{
if (i < 15) I_FatalError("%s: Incorrect PLU size", PLU[i]);
else continue;
}
lookups.setTable(i, (const uint8_t*)data.GetMem());
lookups.setTable(i, data.bytes());
}
lookups.setFadeColor(1, 255, 255, 255);

2
source/games/blood/src/sound.cpp
View file

@ -76,7 +76,7 @@ static void S_AddBloodSFX(int lumpnum)
}
auto sfxlump = fileSystem.ReadFile(lumpnum);
SFX* sfx = (SFX*)sfxlump.GetMem();
SFX* sfx = (SFX*)sfxlump.data();
ByteSwapSFX(sfx);
FStringf rawname("%s.raw", sfx->rawName);

2
source/games/exhumed/src/sound.cpp
View file

@ -177,7 +177,7 @@ int LoadSound(const char* name)
{
auto check = fileSystem.ReadFile(lump);
bool loops = false;
if (check.GetSize() > 26 && check.GetString()[26] == 6 && !memcmp("Creative Voice File", check.GetString(), 19))
if (check.size() > 26 && check.string()[26] == 6 && !memcmp("Creative Voice File", check.string(), 19))
{
// This game uses the actual loop point information in the sound data as its only means to check if a sound is looped.
loops = true;

2
source/games/sw/src/scrip2.cpp
View file

@ -78,7 +78,7 @@ TArray<uint8_t> LoadScriptFile(const char *filename)
return ret;
}
ret.Resize(fp.GetLength() + 1);
ret.Resize((unsigned)fp.GetLength() + 1);
if (fp.Read(ret.Data(), fp.GetLength()) < fp.GetLength())
{
scriptline = 1;