zdoom/raze
0
0
Fork 0
mirror of https://github.com/ZDoom/Raze.git synced 2025-05-30 00:41:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

filesystem update from GZDoom.

Browse source
This commit is contained in:
Christoph Oelckers 2023-12-17 12:48:03 +01:00
parent 84e10beab4
commit 36930d44bd
76 changed files with 3693 additions and 1914 deletions
Download patch file Download diff file Expand all files Collapse all files

63
source/common/filesystem/include/fs_decompress.h Normal file
View file

@ -0,0 +1,63 @@
#pragma once
#include "fs_files.h"
namespace FileSys {
// Zip compression methods, extended by some internal types to be passed to OpenDecompressor
enum ECompressionMethod
{
METHOD_STORED = 0,
METHOD_SHRINK = 1,
METHOD_IMPLODE = 6,
METHOD_DEFLATE = 8,
METHOD_BZIP2 = 12,
METHOD_LZMA = 14,
METHOD_XZ = 95,
METHOD_PPMD = 98,
METHOD_LZSS = 1337, // not used in Zips - this is for Console Doom compression
METHOD_ZLIB = 1338, // Zlib stream with header, used by compressed nodes.
METHOD_RFFCRYPT = 1339, // not actual compression but can be put in here to make handling easier.
METHOD_IMPLODE_MIN = 1000, // having discrete types for these avoids keeping around the GPFlags word in Zips.
METHOD_IMPLODE_0 = 1000,
METHOD_IMPLODE_2 = 1002,
METHOD_IMPLODE_4 = 1004,
METHOD_IMPLODE_6 = 1006,
METHOD_IMPLODE_MAX = 1006,
METHOD_INVALID = 0x7fff,
METHOD_TRANSFEROWNER = 0x8000,
};
enum EDecompressFlags
{
DCF_TRANSFEROWNER = 1,
DCF_SEEKABLE = 2,
DCF_EXCEPTIONS = 4,
DCF_CACHED = 8,
};
bool OpenDecompressor(FileReader& self, FileReader &parent, FileReader::Size length, int method, int flags = 0); // creates a decompressor stream. 'seekable' uses a buffered version so that the Seek and Tell methods can be used.
// This holds a compresed Zip entry with all needed info to decompress it.
struct FCompressedBuffer
{
size_t mSize;
size_t mCompressedSize;
int mMethod;
unsigned mCRC32;
char* mBuffer;
const char* filename;
bool Decompress(char* destbuffer);
void Clean()
{
mSize = mCompressedSize = 0;
if (mBuffer != nullptr)
{
delete[] mBuffer;
mBuffer = nullptr;
}
}
};
}

90
source/common/filesystem/include/fs_files.h
View file

@ -40,12 +40,11 @@
#include <stdio.h>
#include <stdint.h>
#include <stdarg.h>
#include <string.h>
#include <functional>
#include <vector>
#include "fs_swap.h"
#include "tarray.h"
namespace FileSys {
class FileSystemException : public std::exception
@ -72,26 +71,10 @@ public:
}
};
// Zip compression methods, extended by some internal types to be passed to OpenDecompressor
enum
{
METHOD_STORED = 0,
METHOD_SHRINK = 1,
METHOD_IMPLODE = 6,
METHOD_DEFLATE = 8,
METHOD_BZIP2 = 12,
METHOD_LZMA = 14,
METHOD_XZ = 95,
METHOD_PPMD = 98,
METHOD_LZSS = 1337, // not used in Zips - this is for Console Doom compression
METHOD_ZLIB = 1338, // Zlib stream with header, used by compressed nodes.
METHOD_TRANSFEROWNER = 0x8000,
};
class FileReader;
// an opaque memory buffer to the file's content. Can either own the memory or just point to an external buffer.
class ResourceData
class FileData
{
void* memory;
size_t length;
@ -99,13 +82,29 @@ class ResourceData
public:
using value_type = uint8_t;
ResourceData() { memory = nullptr; length = 0; owned = true; }
FileData() { memory = nullptr; length = 0; owned = true; }
FileData(const void* memory_, size_t len, bool own = true)
{
length = len;
if (own)
{
length = len;
memory = allocate(len);
if (memory_) memcpy(memory, memory_, len);
}
else
{
memory = (void*)memory_;
owned = false;
}
}
uint8_t* writable() const { return owned? (uint8_t*)memory : nullptr; }
const void* data() const { return memory; }
size_t size() const { return length; }
const char* string() const { return (const char*)memory; }
const uint8_t* bytes() const { return (const uint8_t*)memory; }
ResourceData& operator = (const ResourceData& copy)
FileData& operator = (const FileData& copy)
{
if (owned && memory) free(memory);
length = copy.length;
@ -119,7 +118,7 @@ public:
return *this;
}
ResourceData& operator = (ResourceData&& copy) noexcept
FileData& operator = (FileData&& copy) noexcept
{
if (owned && memory) free(memory);
length = copy.length;
@ -131,13 +130,13 @@ public:
return *this;
}
ResourceData(const ResourceData& copy)
FileData(const FileData& copy)
{
memory = nullptr;
*this = copy;
}
~ResourceData()
~FileData()
{
if (owned && memory) free(memory);
}
@ -168,6 +167,7 @@ public:
};
class FileReaderInterface
{
public:
@ -181,12 +181,8 @@ public:
ptrdiff_t GetLength () const { return Length; }
};
struct FResourceLump;
class FileReader
{
friend struct FResourceLump; // needs access to the private constructor.
FileReaderInterface *mReader = nullptr;
FileReader(const FileReader &r) = delete;
@ -210,13 +206,13 @@ public:
FileReader() {}
FileReader(FileReader &&r)
FileReader(FileReader &&r) noexcept
{
mReader = r.mReader;
r.mReader = nullptr;
}
FileReader& operator =(FileReader &&r)
FileReader& operator =(FileReader &&r) noexcept
{
Close();
mReader = r.mReader;
@ -252,11 +248,7 @@ public:
bool OpenFile(const char *filename, Size start = 0, Size length = -1, bool buffered = false);
bool OpenFilePart(FileReader &parent, Size start, Size length);
bool OpenMemory(const void *mem, Size length); // read directly from the buffer
bool OpenMemoryArray(const void *mem, Size length); // read from a copy of the buffer.
bool OpenMemoryArray(std::vector<uint8_t>& data); // take the given array
bool OpenMemoryArray(ResourceData& data); // take the given array
bool OpenMemoryArray(std::function<bool(std::vector<uint8_t>&)> getter); // read contents to a buffer and return a reader to it
bool OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions = false); // creates a decompressor stream. 'seekable' uses a buffered version so that the Seek and Tell methods can be used.
bool OpenMemoryArray(FileData& data); // take the given array
Size Tell() const
{
@ -273,36 +265,14 @@ public:
return mReader->Read(buffer, len);
}
ResourceData Read(size_t len)
{
ResourceData buffer;
if (len > 0)
{
Size length = mReader->Read(buffer.allocate(len), len);
if ((size_t)length < len) buffer.allocate(length);
}
return buffer;
}
FileData Read(size_t len);
FileData ReadPadded(size_t padding);
ResourceData Read()
FileData Read()
{
return Read(GetLength());
}
ResourceData ReadPadded(size_t padding)
{
auto len = GetLength();
ResourceData buffer;
if (len > 0)
{
auto p = (char*)buffer.allocate(len + padding);
Size length = mReader->Read(p, len);
if (length < len) buffer.clear();
else memset(p + len, 0, padding);
}
return buffer;
}
char *Gets(char *strbuf, Size len)
{

53
source/common/filesystem/include/fs_filesystem.h
View file

@ -22,42 +22,6 @@ union LumpShortName
};
// A lump in memory.
class FileData
{
public:
FileData() { lump = nullptr; }
const void *GetMem () { return lump->Cache; }
size_t GetSize () { return lump->LumpSize; }
const char* GetString () const { return (const char*)lump->Cache; }
const uint8_t* GetBytes() const { return (const uint8_t*)lump->Cache; }
FileData& operator = (const FileData& copy) = delete;
FileData(const FileData& copy)
{
lump = copy.lump;
lump->Lock();
}
~FileData()
{
if (lump) lump->Unlock();
}
private:
FileData(FResourceLump* nlump)
{
lump = nlump;
if (lump) lump->Lock();
}
FResourceLump* lump;
friend class FileSystem;
};
struct FolderEntry
{
const char *name;
@ -129,9 +93,19 @@ public:
FileData ReadFile (const char *name) { return ReadFile (GetNumForName (name)); }
FileData ReadFileFullName(const char* name) { return ReadFile(GetNumForFullName(name)); }
FileReader OpenFileReader(int lump); // opens a reader that redirects to the containing file's one.
FileReader ReopenFileReader(int lump, bool alwayscache = false); // opens an independent reader.
FileReader OpenFileReader(int lump, int readertype, int readerflags); // opens a reader that redirects to the containing file's one.
FileReader OpenFileReader(const char* name);
FileReader ReopenFileReader(const char* name, bool alwayscache = false);
FileReader OpenFileReader(int lump)
{
return OpenFileReader(lump, READER_SHARED, READERFLAG_SEEKABLE);
}
FileReader ReopenFileReader(int lump, bool alwayscache = false)
{
return OpenFileReader(lump, alwayscache ? READER_CACHED : READER_NEW, READERFLAG_SEEKABLE);
}
int FindLump (const char *name, int *lastlump, bool anyns=false); // [RH] Find lumps with duplication
int FindLumpMulti (const char **names, int *lastlump, bool anyns = false, int *nameindex = NULL); // same with multiple possible names
@ -145,8 +119,7 @@ public:
static uint32_t LumpNameHash (const char *name); // [RH] Create hash key from an 8-char name
int FileLength (int lump) const;
int GetFileOffset (int lump); // [RH] Returns offset of lump in the wadfile
ptrdiff_t FileLength (int lump) const;
int GetFileFlags (int lump); // Return the flags for this lump
const char* GetFileShortName(int lump) const;
const char *GetFileFullName (int lump, bool returnshort = true) const; // [RH] Returns the lump's full name

1
source/common/filesystem/include/fs_findfile.h
View file

@ -23,7 +23,6 @@ using FileList = std::vector<FileListEntry>;
struct FCompressedBuffer;
bool ScanDirectory(std::vector<FileListEntry>& list, const char* dirpath, const char* match, bool nosubdir = false, bool readhidden = false);
bool WriteZip(const char* filename, const FCompressedBuffer* content, size_t contentcount);
bool FS_DirEntryExists(const char* pathname, bool* isdir);
inline void FixPathSeparator(char* path)

182
source/common/filesystem/include/resourcefile.h
View file

@ -7,6 +7,7 @@
#include <vector>
#include <string>
#include "fs_files.h"
#include "fs_decompress.h"
namespace FileSys {
@ -18,7 +19,6 @@ void strReplace(std::string& str, const char* from, const char* to);
struct LumpFilterInfo
{
std::vector<std::string> gameTypeFilter; // this can contain multiple entries
std::string dotFilter;
// The following are for checking if the root directory of a zip can be removed.
std::vector<std::string> reservedFolders;
@ -78,125 +78,69 @@ typedef enum {
enum ELumpFlags
{
LUMPF_MAYBEFLAT = 1, // might be a flat outside F_START/END
LUMPF_FULLPATH = 2, // contains a full path. This will trigger extended namespace checks when looking up short names.
LUMPF_EMBEDDED = 4, // marks an embedded resource file for later processing.
LUMPF_SHORTNAME = 8, // the stored name is a short extension-less name
LUMPF_COMPRESSED = 16, // compressed or encrypted, i.e. cannot be read with the container file's reader.
RESFF_MAYBEFLAT = 1, // might be a flat inside a WAD outside F_START/END
RESFF_FULLPATH = 2, // contains a full path. This will trigger extended namespace checks when looking up short names.
RESFF_EMBEDDED = 4, // marks an embedded resource file for later processing.
RESFF_SHORTNAME = 8, // the stored name is a short extension-less name
RESFF_COMPRESSED = 16, // compressed or encrypted, i.e. cannot be read with the container file's reader.
RESFF_NEEDFILESTART = 32, // The real position is not known yet and needs to be calculated on access
};
// This holds a compresed Zip entry with all needed info to decompress it.
struct FCompressedBuffer
enum EReaderType
{
size_t mSize;
size_t mCompressedSize;
int mMethod;
int mZipFlags;
unsigned mCRC32;
char *mBuffer;
const char* filename;
bool Decompress(char *destbuffer);
void Clean()
{
mSize = mCompressedSize = 0;
if (mBuffer != nullptr)
{
delete[] mBuffer;
mBuffer = nullptr;
}
}
READER_SHARED = 0, // returns a view into the parent's reader.
READER_NEW = 1, // opens a new file handle
READER_CACHED = 2, // returns a MemoryArrayReader
READERFLAG_SEEKABLE = 1 // ensure the reader is seekable.
};
struct FResourceLump
struct FResourceEntry
{
protected:
friend class FResourceFile;
friend class FWadFile; // this still needs direct access.
friend class FileData;
friend class FileSystem;
friend class FLumpFile;
friend class FLumpReader;
friend class FGrpFile;
friend class F7ZFile;
friend class FSSIFile;
friend class FWHResFile;
friend class FZipFile;
friend class FPakFile;
friend class FRFFFile;
friend class FDirectory;
friend int lumpcmp(const void* a, const void* b);
int LumpSize;
int RefCount;
//protected:
const char* FullName;
//public:
uint8_t Flags;
char * Cache;
FResourceFile * Owner;
public:
FResourceLump()
{
Cache = NULL;
Owner = NULL;
Flags = 0;
RefCount = 0;
FullName = "";
LumpSize = 0;
}
virtual ~FResourceLump();
protected:
virtual FileReader *GetReader();
virtual FileReader NewReader();
virtual int GetFileOffset() { return -1; }
virtual int GetIndexNum() const { return -1; }
virtual int GetNamespace() const { return 0; }
void LumpNameSetup(const char* iname, StringPool* allocator);
void CheckEmbedded(LumpFilterInfo* lfi);
virtual FCompressedBuffer GetRawData();
void *Lock(); // validates the cache and increases the refcount.
int Unlock(); // decreases the refcount and frees the buffer
unsigned Size() const{ return LumpSize; }
int LockCount() const { return RefCount; }
const char* getName() { return FullName; }
void clearName() { FullName = ""; }
protected:
virtual int FillCache() { return -1; }
size_t Length;
size_t CompressedSize;
const char* FileName;
size_t Position;
int ResourceID;
uint32_t CRC32;
uint16_t Flags;
uint16_t Method;
int16_t Namespace;
};
void SetMainThread();
class FResourceFile
{
public:
FResourceFile(const char* filename, StringPool* sp);
FResourceFile(const char* filename, FileReader& r, StringPool* sp);
const char* NormalizeFileName(const char* fn, int fallbackcp = 0);
FResourceEntry* AllocateEntries(int count);
void GenerateHash();
void PostProcessArchive(LumpFilterInfo* filter);
protected:
FileReader Reader;
const char* FileName;
FResourceEntry* Entries = nullptr;
uint32_t NumLumps;
char Hash[48];
StringPool* stringpool;
FResourceFile(const char *filename, StringPool* sp);
FResourceFile(const char *filename, FileReader &r, StringPool* sp);
// for archives that can contain directories
void GenerateHash();
void PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterInfo *filter);
virtual void SetEntryAddress(uint32_t entry)
{
Entries[entry].Flags &= ~RESFF_NEEDFILESTART;
}
bool IsFileInFolder(const char* const resPath);
void CheckEmbedded(uint32_t entry, LumpFilterInfo* lfi);
private:
uint32_t FirstLump;
int FilterLumps(const std::string& filtername, void *lumps, size_t lumpsize, uint32_t max);
int FilterLumpsByGameType(LumpFilterInfo *filter, void *lumps, size_t lumpsize, uint32_t max);
bool FindPrefixRange(const char* filter, void *lumps, size_t lumpsize, uint32_t max, uint32_t &start, uint32_t &end);
void JunkLeftoverFilters(void *lumps, size_t lumpsize, uint32_t max);
int FilterLumps(const std::string& filtername, uint32_t max);
bool FindPrefixRange(const char* filter, uint32_t max, uint32_t &start, uint32_t &end);
void JunkLeftoverFilters(uint32_t max);
void FindCommonFolder(LumpFilterInfo* filter);
static FResourceFile *DoOpenResourceFile(const char *filename, FileReader &file, bool containeronly, LumpFilterInfo* filter, FileSystemMessageFunc Printf, StringPool* sp);
public:
@ -211,47 +155,45 @@ public:
void SetFirstLump(uint32_t f) { FirstLump = f; }
const char* GetHash() const { return Hash; }
virtual FResourceLump *GetLump(int no) = 0;
int EntryCount() const { return NumLumps; }
int FindEntry(const char* name);
size_t Length(int entry)
size_t Length(uint32_t entry)
{
auto l = GetLump(entry);
return l ? l->LumpSize : -1;
return (entry < NumLumps) ? Entries[entry].Length : 0;
}
size_t Offset(uint32_t entry)
{
return (entry < NumLumps) ? Entries[entry].Position : 0;
}
FileReader GetEntryReader(int entry, bool newreader = true)
// default is the safest reader type.
virtual FileReader GetEntryReader(uint32_t entry, int readertype = READER_NEW, int flags = READERFLAG_SEEKABLE);
int GetEntryFlags(uint32_t entry)
{
auto l = GetLump(entry);
return l ? l->NewReader() : FileReader();
return (entry < NumLumps) ? Entries[entry].Flags : 0;
}
int GetEntryFlags(int entry)
int GetEntryNamespace(uint32_t entry)
{
auto l = GetLump(entry);
return l ? l->Flags : 0;
return (entry < NumLumps) ? Entries[entry].Namespace : ns_hidden;
}
ResourceData Read(int entry)
int GetEntryResourceID(uint32_t entry)
{
auto fr = GetEntryReader(entry, false);
return fr.Read();
return (entry < NumLumps) ? Entries[entry].ResourceID : -1;
}
const char* getName(int entry)
const char* getName(uint32_t entry)
{
auto l = GetLump(entry);
return l ? l->FullName : nullptr;
}
FCompressedBuffer GetRawData(int entry)
{
auto l = GetLump(entry);
if (!l) return {};
return l->GetRawData();
return (entry < NumLumps) ? Entries[entry].FileName : nullptr;
}
virtual FileData Read(int entry);
virtual FCompressedBuffer GetRawData(uint32_t entry);
FileReader Destroy()
{
auto fr = std::move(Reader);

111
source/common/filesystem/include/w_zip.h Normal file
View file

@ -0,0 +1,111 @@
#ifndef __W_ZIP
#define __W_ZIP
#if defined(__GNUC__)
#define FORCE_PACKED __attribute__((__packed__))
#else
#define FORCE_PACKED
#endif
#include <stdint.h>
#pragma pack(1)
// FZipCentralInfo
struct FZipEndOfCentralDirectory
{
uint32_t Magic;
uint16_t DiskNumber;
uint16_t FirstDisk;
uint16_t NumEntries;
uint16_t NumEntriesOnAllDisks;
uint32_t DirectorySize;
uint32_t DirectoryOffset;
uint16_t ZipCommentLength;
} FORCE_PACKED;
struct FZipEndOfCentralDirectory64
{
uint32_t Magic;
uint64_t StructSize;
uint16_t VersionMadeBy;
uint16_t VersionNeeded;
uint32_t DiskNumber;
uint32_t FirstDisk;
uint64_t NumEntries;
uint64_t NumEntriesOnAllDisks;
uint64_t DirectorySize;
uint64_t DirectoryOffset;
uint16_t ZipCommentLength;
} FORCE_PACKED;
// FZipFileInfo
struct FZipCentralDirectoryInfo
{
uint32_t Magic;
uint8_t VersionMadeBy[2];
uint8_t VersionToExtract[2];
uint16_t Flags;
uint16_t Method;
uint16_t ModTime;
uint16_t ModDate;
uint32_t CRC32;
uint32_t CompressedSize32;
uint32_t UncompressedSize32;
uint16_t NameLength;
uint16_t ExtraLength;
uint16_t CommentLength;
uint16_t StartingDiskNumber;
uint16_t InternalAttributes;
uint32_t ExternalAttributes;
uint32_t LocalHeaderOffset32;
// file name and other variable length info follows
} FORCE_PACKED;
struct FZipCentralDirectoryInfo64BitExt
{
uint16_t Type;
uint16_t Length;
uint64_t UncompressedSize;
uint64_t CompressedSize;
uint64_t LocalHeaderOffset;
uint32_t DiskNo;
} FORCE_PACKED;
// FZipLocalHeader
struct FZipLocalFileHeader
{
uint32_t Magic;
uint8_t VersionToExtract[2];
uint16_t Flags;
uint16_t Method;
uint16_t ModTime;
uint16_t ModDate;
uint32_t CRC32;
uint32_t CompressedSize;
uint32_t UncompressedSize;
uint16_t NameLength;
uint16_t ExtraLength;
// file name and other variable length info follows
} FORCE_PACKED;
#pragma pack()
#ifndef MAKE_ID
#ifndef __BIG_ENDIAN__
#define MAKE_ID(a,b,c,d) ((uint32_t)((a)|((b)<<8)|((c)<<16)|((d)<<24)))
#else
#define MAKE_ID(a,b,c,d) ((uint32_t)((d)|((c)<<8)|((b)<<16)|((a)<<24)))
#endif
#endif
#define ZIP_LOCALFILE MAKE_ID('P','K',3,4)
#define ZIP_CENTRALFILE MAKE_ID('P','K',1,2)
#define ZIP_ENDOFDIR MAKE_ID('P','K',5,6)
// File header flags.
#define ZF_ENCRYPTED 0x1
#endif