- implemented savegame compression

Unfortunately necessary because Ion Fury savegames store 120 GB(!!) of data, mostly zeros.

Unlike the old method, this compresses the entire savegame as one block using a ZLib stream so it should be a lot more efficient now.
This commit is contained in:
Christoph Oelckers 2019-11-08 01:36:32 +01:00
parent a40be954f1
commit 4fc56203c2
5 changed files with 256 additions and 55 deletions

View file

@ -68,7 +68,7 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, int L
case METHOD_LZMA:
{
FileReader frz;
if (frz.OpenDecompressor(Reader, LumpSize, Method, false, [](const char* err) { I_Error("%s", err); }))
if (frz.OpenDecompressor(Reader, LumpSize, Method, false, nullptr))
{
frz.Read(Cache, LumpSize);
}

View file

@ -56,6 +56,7 @@ enum
METHOD_PPMD = 98,
METHOD_LZSS = 1337, // not used in Zips - this is for Console Doom compression
METHOD_ZLIB = 1338, // Zlib stream with header, used by compressed nodes.
METHOD_TRANSFEROWNER = 0x8000,
};
class FileReaderInterface
@ -85,6 +86,11 @@ public:
{
ErrorCallback = cb;
}
void SetOwnsReader();
protected:
FileReader *File = nullptr;
FileReader OwnedFile;
};
class MemoryReader : public FileReaderInterface
@ -316,7 +322,7 @@ public:
}
virtual ~FileWriter()
{
if (File != NULL) fclose(File);
Close();
}
static FileWriter *Open(const char *filename);
@ -325,7 +331,7 @@ public:
virtual long Tell();
virtual long Seek(long offset, int mode);
size_t Printf(const char *fmt, ...) GCCPRINTF(2,3);
void Close()
virtual void Close()
{
if (File != NULL) fclose(File);
File = nullptr;
@ -351,4 +357,23 @@ public:
TArray<unsigned char>&& TakeBuffer() { return std::move(mBuffer); }
};
class CompressedFileWriter : public FileWriter
{
FileWriter *target;
struct z_stream_s *zipstream;
uint8_t outbuf[1024];
size_t compressedSize;
bool ownsWriter;
size_t WriteBlock(const void *buffer, size_t bytes);
public:
CompressedFileWriter(FileWriter *wr, bool transfer = false);
CompressedFileWriter(FILE *wr);
~CompressedFileWriter() { Close(); }
virtual size_t Write(const void *buffer, size_t len) override;
virtual void Close() override;
};
#endif

View file

@ -63,7 +63,7 @@ void DecompressorBase::DecompressionError(const char *error, ...) const
va_end(argptr);
if (ErrorCallback != nullptr) ErrorCallback(errortext);
else std::terminate();
else throw std::runtime_error(errortext);
}
long DecompressorBase::Tell () const
@ -82,6 +82,12 @@ char *DecompressorBase::Gets(char *strbuf, int len)
return nullptr;
}
void DecompressorBase::SetOwnsReader()
{
OwnedFile = std::move(*File);
File = &OwnedFile;
}
//
// M_ZlibError
//
@ -125,17 +131,17 @@ class DecompressorZ : public DecompressorBase
{
enum { BUFF_SIZE = 4096 };
FileReader &File;
bool SawEOF;
z_stream Stream;
uint8_t InBuff[BUFF_SIZE];
public:
DecompressorZ (FileReader &file, bool zip, const std::function<void(const char*)>& cb)
: File(file), SawEOF(false)
DecompressorZ (FileReader *file, bool zip, const std::function<void(const char*)>& cb)
: SawEOF(false)
{
int err;
File = file;
SetErrorCallback(cb);
FillBuffer ();
@ -187,7 +193,7 @@ public:
void FillBuffer ()
{
auto numread = File.Read (InBuff, BUFF_SIZE);
auto numread = File->Read (InBuff, BUFF_SIZE);
if (numread < BUFF_SIZE)
{
@ -216,17 +222,17 @@ class DecompressorBZ2 : public DecompressorBase
{
enum { BUFF_SIZE = 4096 };
FileReader &File;
bool SawEOF;
bz_stream Stream;
uint8_t InBuff[BUFF_SIZE];
public:
DecompressorBZ2 (FileReader &file, const std::function<void(const char*)>& cb)
: File(file), SawEOF(false)
DecompressorBZ2 (FileReader *file, const std::function<void(const char*)>& cb)
: SawEOF(false)
{
int err;
File = file;
SetErrorCallback(cb);
stupidGlobal = this;
FillBuffer ();
@ -281,7 +287,7 @@ public:
void FillBuffer ()
{
auto numread = File.Read(InBuff, BUFF_SIZE);
auto numread = File->Read(InBuff, BUFF_SIZE);
if (numread < BUFF_SIZE)
{
@ -325,7 +331,6 @@ class DecompressorLZMA : public DecompressorBase
{
enum { BUFF_SIZE = 4096 };
FileReader &File;
bool SawEOF;
CLzmaDec Stream;
size_t Size;
@ -335,18 +340,19 @@ class DecompressorLZMA : public DecompressorBase
public:
DecompressorLZMA (FileReader &file, size_t uncompressed_size, const std::function<void(const char*)>& cb)
: File(file), SawEOF(false)
DecompressorLZMA (FileReader *file, size_t uncompressed_size, const std::function<void(const char*)>& cb)
: SawEOF(false)
{
uint8_t header[4 + LZMA_PROPS_SIZE];
int err;
File = file;
SetErrorCallback(cb);
Size = uncompressed_size;
OutProcessed = 0;
// Read zip LZMA properties header
if (File.Read(header, sizeof(header)) < (long)sizeof(header))
if (File->Read(header, sizeof(header)) < (long)sizeof(header))
{
DecompressionError("DecompressorLZMA: File too short\n");
}
@ -423,7 +429,7 @@ public:
void FillBuffer ()
{
auto numread = File.Read(InBuff, BUFF_SIZE);
auto numread = File->Read(InBuff, BUFF_SIZE);
if (numread < BUFF_SIZE)
{
@ -445,7 +451,6 @@ class DecompressorLZSS : public DecompressorBase
{
enum { BUFF_SIZE = 4096, WINDOW_SIZE = 4096, INTERNAL_BUFFER_SIZE = 128 };
FileReader &File;
bool SawEOF;
uint8_t InBuff[BUFF_SIZE];
@ -476,7 +481,7 @@ class DecompressorLZSS : public DecompressorBase
if(Stream.AvailIn)
memmove(InBuff, Stream.In, Stream.AvailIn);
auto numread = File.Read(InBuff+Stream.AvailIn, BUFF_SIZE-Stream.AvailIn);
auto numread = File->Read(InBuff+Stream.AvailIn, BUFF_SIZE-Stream.AvailIn);
if (numread < BUFF_SIZE)
{
@ -563,8 +568,9 @@ class DecompressorLZSS : public DecompressorBase
}
public:
DecompressorLZSS(FileReader &file, const std::function<void(const char*)>& cb) : File(file), SawEOF(false)
DecompressorLZSS(FileReader *file, const std::function<void(const char*)>& cb) : File(file), SawEOF(false)
{
File = file;
SetErrorCallback(cb);
Stream.State = STREAM_EMPTY;
Stream.WindowData = Stream.InternalBuffer = Stream.Window+WINDOW_SIZE;
@ -629,29 +635,35 @@ public:
bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, const std::function<void(const char*)>& cb)
{
DecompressorBase *dec = nullptr;
switch (method)
FileReader *p = &parent;
switch (method & ~METHOD_TRANSFEROWNER)
{
case METHOD_DEFLATE:
case METHOD_ZLIB:
dec = new DecompressorZ(parent, method == METHOD_DEFLATE, cb);
dec = new DecompressorZ(p, method == METHOD_DEFLATE, cb);
break;
case METHOD_BZIP2:
dec = new DecompressorBZ2(parent, cb);
dec = new DecompressorBZ2(p, cb);
break;
case METHOD_LZMA:
dec = new DecompressorLZMA(parent, length, cb);
dec = new DecompressorLZMA(p, length, cb);
break;
case METHOD_LZSS:
dec = new DecompressorLZSS(parent, cb);
dec = new DecompressorLZSS(p, cb);
break;
// todo: METHOD_IMPLODE, METHOD_SHRINK
default:
return false;
}
if (method & METHOD_TRANSFEROWNER)
{
dec->SetOwnsReader();
}
dec->Length = (long)length;
if (!seekable)
{
@ -666,3 +678,117 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
return false;
}
}
//==========================================================================
//
//
//
//==========================================================================
CompressedFileWriter::CompressedFileWriter(FileWriter *targ, bool transfer)
{
target = targ;
zipstream = new z_stream;
compressedSize = 0;
zipstream->next_in = Z_NULL;
zipstream->avail_in = 0;
zipstream->zalloc = Z_NULL;
zipstream->zfree = Z_NULL;
int err = deflateInit2 (zipstream, Z_BEST_COMPRESSION, Z_DEFLATED, -MAX_WBITS, 8, Z_DEFAULT_STRATEGY);
if (err != Z_OK)
{
delete zipstream;
zipstream = nullptr;
return;
}
zipstream->next_out = outbuf;
zipstream->avail_out = sizeof(outbuf);
ownsWriter = transfer;
}
//==========================================================================
//
//
//
//==========================================================================
CompressedFileWriter::CompressedFileWriter(FILE *targ)
: CompressedFileWriter(new FileWriter(targ), true)
{
}
//==========================================================================
//
//
//
//==========================================================================
size_t CompressedFileWriter::Write(const void *buffer, size_t bytes)
{
size_t wrote = 0;
size_t towrite = bytes;
zipstream->next_in = (Bytef *)buffer;
while (bytes > 0)
{
auto chunk = std::min(towrite, (size_t)0x40000000);
zipstream->avail_in = chunk;
buffer = ((char*)buffer) + chunk;
towrite -= chunk;
while (zipstream->avail_in != 0)
{
if (zipstream->avail_out == 0)
{
zipstream->next_out = outbuf;
zipstream->avail_out = 1024;
wrote += 1024;
target->Write(outbuf, 1024);
}
deflate (zipstream, Z_NO_FLUSH);
}
}
compressedSize += wrote;
return bytes;
}
//==========================================================================
//
//
//
//==========================================================================
void CompressedFileWriter::Close()
{
if (!zipstream) return;
// Flush the zlib stream buffer.
for (bool done = false;;)
{
auto len = sizeof(outbuf) - zipstream->avail_out;
if (len != 0)
{
compressedSize += len;
target->Write(outbuf, len);
zipstream->next_out = outbuf;
zipstream->avail_out = sizeof(outbuf);
}
if (done)
{
break;
}
auto err = deflate (zipstream, Z_FINISH);
done = stream.avail_out != 0 || err == Z_STREAM_END;
if (err != Z_STREAM_END && err != Z_OK)
{
break;
}
}
deflateEnd (zipstream);
delete zipstream;
zipstream = nullptr;
}

View file

@ -152,14 +152,36 @@ uint16_t g_nummenusaves;
static menusave_t * g_internalsaves;
static uint16_t g_numinternalsaves;
static FileReader OpenSavegame(const char *fn)
{
auto file = fopenFileReader(fn, 0);
if (!file.isOpen())
return file;
char buffer[13];
file.Read(buffer, 13);
if (memcmp(buffer, "DEMOLITION_ED", 13))
return FileReader();
FileReader fr;
try
{
fr.OpenDecompressor(file, file.GetLength()-13, METHOD_DEFLATE, false, nullptr);
}
catch(std::runtime_error & err)
{
Printf("%s: %s\n", fn, err.what());
}
return fr;
}
static void ReadSaveGameHeaders_CACHE1D(TArray<FString> &saves)
{
savehead_t h;
for (auto &save : saves)
for (FString &save : saves)
{
char const * fn = save;
auto fil = fopenFileReader(fn, 0);
auto fil = OpenSavegame(save);
if (!fil.isOpen())
continue;
@ -176,7 +198,7 @@ static void ReadSaveGameHeaders_CACHE1D(TArray<FString> &saves)
{
if (FURY)
{
FStringf extfn("%s.ext", fn);
FStringf extfn("%s.ext", save.GetChars());
auto extfil = fopenFileReader(extfn, 0);
if (extfil.isOpen())
{
@ -191,7 +213,7 @@ static void ReadSaveGameHeaders_CACHE1D(TArray<FString> &saves)
msv.isAutoSave = h.isAutoSave();
strncpy(msv.brief.path, fn, ARRAY_SIZE(msv.brief.path));
strncpy(msv.brief.path, save.GetChars(), ARRAY_SIZE(msv.brief.path));
++g_numinternalsaves;
if (k >= 0 && h.savename[0] != '\0')
@ -293,7 +315,7 @@ void ReadSaveGameHeaders(void)
int32_t G_LoadSaveHeaderNew(char const *fn, savehead_t *saveh)
{
auto fil = fopenFileReader(fn, 0);
auto fil = OpenSavegame(fn);
if (!fil.isOpen())
return -1;
@ -342,7 +364,7 @@ int32_t G_LoadPlayer(savebrief_t & sv)
int level = -1;
int skill = -1;
auto fil = fopenFileReader(sv.path, 0);
auto fil = OpenSavegame(sv.path);
if (fil.isOpen())
{
@ -569,7 +591,7 @@ int32_t G_LoadPlayer(savebrief_t & sv)
return 0;
}
auto fil = fopenFileReader(sv.path, 0);
auto fil = OpenSavegame(sv.path);
if (!fil.isOpen())
return -1;
@ -762,11 +784,10 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
Bstrcpy(sv.path, fn + (fn.Len() - (ARRAY_SIZE(SaveName) - 1)));
}
FileWriter fw(fil);
if (!fil)
{
OSD_Printf("G_SavePlayer: failed opening \"%s\" for writing: %s\n",
fn, strerror(errno));
fn.GetChars(), strerror(errno));
ready2send = 1;
Net_WaitForServer();
@ -776,6 +797,9 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
}
else
{
fwrite("DEMOLITION_ED", 13, 1, fil);
CompressedFileWriter fw(fil);
sv.isExt = 0;
// temporary hack
@ -792,7 +816,7 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
if (!g_netServer && ud.multimode < 2)
{
OSD_Printf("Saved: %s\n", fn);
OSD_Printf("Saved: %s\n", fn.GetChars());
strcpy(apStrings[QUOTE_RESERVED4], "Game Saved");
P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps);
}

View file

@ -147,14 +147,37 @@ uint16_t g_nummenusaves;
static menusave_t * g_internalsaves;
static uint16_t g_numinternalsaves;
static FileReader OpenSavegame(const char *fn)
{
auto file = fopenFileReader(fn, 0);
if (!file.isOpen())
return file;
char buffer[13];
file.Read(buffer, 13);
if (memcmp(buffer, "DEMOLITION_RN", 13))
return FileReader();
FileReader fr;
try
{
fr.OpenDecompressor(file, file.GetLength()-13, METHOD_DEFLATE|METHOD_TRANSFEROWNER, false, nullptr);
}
catch(std::runtime_error & err)
{
Printf("%s: %s\n", fn, err.what());
}
return fr;
}
static void ReadSaveGameHeaders_CACHE1D(TArray<FString>& saves)
{
savehead_t h;
for (auto& save : saves)
for (FString& save : saves)
{
char const* fn = save;
auto fil = fopenFileReader(fn, 0);
auto fil = OpenSavegame(fn);
if (!fil.isOpen())
continue;
@ -274,7 +297,7 @@ void ReadSaveGameHeaders(void)
int32_t G_LoadSaveHeaderNew(char const *fn, savehead_t *saveh)
{
auto fil = fopenFileReader(fn, 0);
auto fil = OpenSavegame(fn);
if (!fil.isOpen())
return -1;
@ -316,7 +339,7 @@ static int different_user_map;
// XXX: keyboard input 'blocked' after load fail? (at least ESC?)
int32_t G_LoadPlayer(savebrief_t & sv)
{
auto fil = fopenFileReader(sv.path, 0);
auto fil = OpenSavegame(sv.path);
if (!fil.isOpen())
return -1;
@ -504,11 +527,10 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
Bstrcpy(sv.path, fn + (fn.Len() - (ARRAY_SIZE(SaveName) - 1)));
}
FileWriter fw(fil);
if (!fil)
{
OSD_Printf("G_SavePlayer: failed opening \"%s\" for writing: %s\n",
fn, strerror(errno));
fn.GetChars(), strerror(errno));
ready2send = 1;
Net_WaitForEverybody();
@ -518,6 +540,9 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
}
else
{
fwrite("DEMOLITION_RN", 13, 1, fil);
CompressedFileWriter fw(fil);
// temporary hack
ud.user_map = G_HaveUserMap();
@ -527,19 +552,20 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
fw.Close();
if (!g_netServer && ud.multimode < 2)
{
Bstrcpy(apStrings[QUOTE_RESERVED4], "Game Saved");
P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps);
}
ready2send = 1;
Net_WaitForEverybody();
G_RestoreTimers();
ototalclock = totalclock;
return 0;
if (!g_netServer && ud.multimode < 2)
{
OSD_Printf("Saved: %s\n", fn.GetChars());
Bstrcpy(apStrings[QUOTE_RESERVED4], "Game Saved");
P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps);
}
ready2send = 1;
Net_WaitForEverybody();
G_RestoreTimers();
ototalclock = totalclock;
return 0;
}
}