- implemented savegame compression

Unfortunately necessary because Ion Fury savegames store 120 GB(!!) of data, mostly zeros.

Unlike the old method, this compresses the entire savegame as one block using a ZLib stream so it should be a lot more efficient now.
This commit is contained in:
Christoph Oelckers 2019-11-08 01:36:32 +01:00
parent a40be954f1
commit 4fc56203c2
5 changed files with 256 additions and 55 deletions

View file

@ -68,7 +68,7 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, int L
case METHOD_LZMA: case METHOD_LZMA:
{ {
FileReader frz; FileReader frz;
if (frz.OpenDecompressor(Reader, LumpSize, Method, false, [](const char* err) { I_Error("%s", err); })) if (frz.OpenDecompressor(Reader, LumpSize, Method, false, nullptr))
{ {
frz.Read(Cache, LumpSize); frz.Read(Cache, LumpSize);
} }

View file

@ -56,6 +56,7 @@ enum
METHOD_PPMD = 98, METHOD_PPMD = 98,
METHOD_LZSS = 1337, // not used in Zips - this is for Console Doom compression METHOD_LZSS = 1337, // not used in Zips - this is for Console Doom compression
METHOD_ZLIB = 1338, // Zlib stream with header, used by compressed nodes. METHOD_ZLIB = 1338, // Zlib stream with header, used by compressed nodes.
METHOD_TRANSFEROWNER = 0x8000,
}; };
class FileReaderInterface class FileReaderInterface
@ -85,6 +86,11 @@ public:
{ {
ErrorCallback = cb; ErrorCallback = cb;
} }
void SetOwnsReader();
protected:
FileReader *File = nullptr;
FileReader OwnedFile;
}; };
class MemoryReader : public FileReaderInterface class MemoryReader : public FileReaderInterface
@ -316,7 +322,7 @@ public:
} }
virtual ~FileWriter() virtual ~FileWriter()
{ {
if (File != NULL) fclose(File); Close();
} }
static FileWriter *Open(const char *filename); static FileWriter *Open(const char *filename);
@ -325,7 +331,7 @@ public:
virtual long Tell(); virtual long Tell();
virtual long Seek(long offset, int mode); virtual long Seek(long offset, int mode);
size_t Printf(const char *fmt, ...) GCCPRINTF(2,3); size_t Printf(const char *fmt, ...) GCCPRINTF(2,3);
void Close() virtual void Close()
{ {
if (File != NULL) fclose(File); if (File != NULL) fclose(File);
File = nullptr; File = nullptr;
@ -351,4 +357,23 @@ public:
TArray<unsigned char>&& TakeBuffer() { return std::move(mBuffer); } TArray<unsigned char>&& TakeBuffer() { return std::move(mBuffer); }
}; };
class CompressedFileWriter : public FileWriter
{
FileWriter *target;
struct z_stream_s *zipstream;
uint8_t outbuf[1024];
size_t compressedSize;
bool ownsWriter;
size_t WriteBlock(const void *buffer, size_t bytes);
public:
CompressedFileWriter(FileWriter *wr, bool transfer = false);
CompressedFileWriter(FILE *wr);
~CompressedFileWriter() { Close(); }
virtual size_t Write(const void *buffer, size_t len) override;
virtual void Close() override;
};
#endif #endif

View file

@ -63,7 +63,7 @@ void DecompressorBase::DecompressionError(const char *error, ...) const
va_end(argptr); va_end(argptr);
if (ErrorCallback != nullptr) ErrorCallback(errortext); if (ErrorCallback != nullptr) ErrorCallback(errortext);
else std::terminate(); else throw std::runtime_error(errortext);
} }
long DecompressorBase::Tell () const long DecompressorBase::Tell () const
@ -82,6 +82,12 @@ char *DecompressorBase::Gets(char *strbuf, int len)
return nullptr; return nullptr;
} }
void DecompressorBase::SetOwnsReader()
{
OwnedFile = std::move(*File);
File = &OwnedFile;
}
// //
// M_ZlibError // M_ZlibError
// //
@ -125,17 +131,17 @@ class DecompressorZ : public DecompressorBase
{ {
enum { BUFF_SIZE = 4096 }; enum { BUFF_SIZE = 4096 };
FileReader &File;
bool SawEOF; bool SawEOF;
z_stream Stream; z_stream Stream;
uint8_t InBuff[BUFF_SIZE]; uint8_t InBuff[BUFF_SIZE];
public: public:
DecompressorZ (FileReader &file, bool zip, const std::function<void(const char*)>& cb) DecompressorZ (FileReader *file, bool zip, const std::function<void(const char*)>& cb)
: File(file), SawEOF(false) : SawEOF(false)
{ {
int err; int err;
File = file;
SetErrorCallback(cb); SetErrorCallback(cb);
FillBuffer (); FillBuffer ();
@ -187,7 +193,7 @@ public:
void FillBuffer () void FillBuffer ()
{ {
auto numread = File.Read (InBuff, BUFF_SIZE); auto numread = File->Read (InBuff, BUFF_SIZE);
if (numread < BUFF_SIZE) if (numread < BUFF_SIZE)
{ {
@ -216,17 +222,17 @@ class DecompressorBZ2 : public DecompressorBase
{ {
enum { BUFF_SIZE = 4096 }; enum { BUFF_SIZE = 4096 };
FileReader &File;
bool SawEOF; bool SawEOF;
bz_stream Stream; bz_stream Stream;
uint8_t InBuff[BUFF_SIZE]; uint8_t InBuff[BUFF_SIZE];
public: public:
DecompressorBZ2 (FileReader &file, const std::function<void(const char*)>& cb) DecompressorBZ2 (FileReader *file, const std::function<void(const char*)>& cb)
: File(file), SawEOF(false) : SawEOF(false)
{ {
int err; int err;
File = file;
SetErrorCallback(cb); SetErrorCallback(cb);
stupidGlobal = this; stupidGlobal = this;
FillBuffer (); FillBuffer ();
@ -281,7 +287,7 @@ public:
void FillBuffer () void FillBuffer ()
{ {
auto numread = File.Read(InBuff, BUFF_SIZE); auto numread = File->Read(InBuff, BUFF_SIZE);
if (numread < BUFF_SIZE) if (numread < BUFF_SIZE)
{ {
@ -325,7 +331,6 @@ class DecompressorLZMA : public DecompressorBase
{ {
enum { BUFF_SIZE = 4096 }; enum { BUFF_SIZE = 4096 };
FileReader &File;
bool SawEOF; bool SawEOF;
CLzmaDec Stream; CLzmaDec Stream;
size_t Size; size_t Size;
@ -335,18 +340,19 @@ class DecompressorLZMA : public DecompressorBase
public: public:
DecompressorLZMA (FileReader &file, size_t uncompressed_size, const std::function<void(const char*)>& cb) DecompressorLZMA (FileReader *file, size_t uncompressed_size, const std::function<void(const char*)>& cb)
: File(file), SawEOF(false) : SawEOF(false)
{ {
uint8_t header[4 + LZMA_PROPS_SIZE]; uint8_t header[4 + LZMA_PROPS_SIZE];
int err; int err;
File = file;
SetErrorCallback(cb); SetErrorCallback(cb);
Size = uncompressed_size; Size = uncompressed_size;
OutProcessed = 0; OutProcessed = 0;
// Read zip LZMA properties header // Read zip LZMA properties header
if (File.Read(header, sizeof(header)) < (long)sizeof(header)) if (File->Read(header, sizeof(header)) < (long)sizeof(header))
{ {
DecompressionError("DecompressorLZMA: File too short\n"); DecompressionError("DecompressorLZMA: File too short\n");
} }
@ -423,7 +429,7 @@ public:
void FillBuffer () void FillBuffer ()
{ {
auto numread = File.Read(InBuff, BUFF_SIZE); auto numread = File->Read(InBuff, BUFF_SIZE);
if (numread < BUFF_SIZE) if (numread < BUFF_SIZE)
{ {
@ -445,7 +451,6 @@ class DecompressorLZSS : public DecompressorBase
{ {
enum { BUFF_SIZE = 4096, WINDOW_SIZE = 4096, INTERNAL_BUFFER_SIZE = 128 }; enum { BUFF_SIZE = 4096, WINDOW_SIZE = 4096, INTERNAL_BUFFER_SIZE = 128 };
FileReader &File;
bool SawEOF; bool SawEOF;
uint8_t InBuff[BUFF_SIZE]; uint8_t InBuff[BUFF_SIZE];
@ -476,7 +481,7 @@ class DecompressorLZSS : public DecompressorBase
if(Stream.AvailIn) if(Stream.AvailIn)
memmove(InBuff, Stream.In, Stream.AvailIn); memmove(InBuff, Stream.In, Stream.AvailIn);
auto numread = File.Read(InBuff+Stream.AvailIn, BUFF_SIZE-Stream.AvailIn); auto numread = File->Read(InBuff+Stream.AvailIn, BUFF_SIZE-Stream.AvailIn);
if (numread < BUFF_SIZE) if (numread < BUFF_SIZE)
{ {
@ -563,8 +568,9 @@ class DecompressorLZSS : public DecompressorBase
} }
public: public:
DecompressorLZSS(FileReader &file, const std::function<void(const char*)>& cb) : File(file), SawEOF(false) DecompressorLZSS(FileReader *file, const std::function<void(const char*)>& cb) : File(file), SawEOF(false)
{ {
File = file;
SetErrorCallback(cb); SetErrorCallback(cb);
Stream.State = STREAM_EMPTY; Stream.State = STREAM_EMPTY;
Stream.WindowData = Stream.InternalBuffer = Stream.Window+WINDOW_SIZE; Stream.WindowData = Stream.InternalBuffer = Stream.Window+WINDOW_SIZE;
@ -629,29 +635,35 @@ public:
bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, const std::function<void(const char*)>& cb) bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, const std::function<void(const char*)>& cb)
{ {
DecompressorBase *dec = nullptr; DecompressorBase *dec = nullptr;
switch (method) FileReader *p = &parent;
switch (method & ~METHOD_TRANSFEROWNER)
{ {
case METHOD_DEFLATE: case METHOD_DEFLATE:
case METHOD_ZLIB: case METHOD_ZLIB:
dec = new DecompressorZ(parent, method == METHOD_DEFLATE, cb); dec = new DecompressorZ(p, method == METHOD_DEFLATE, cb);
break; break;
case METHOD_BZIP2: case METHOD_BZIP2:
dec = new DecompressorBZ2(parent, cb); dec = new DecompressorBZ2(p, cb);
break; break;
case METHOD_LZMA: case METHOD_LZMA:
dec = new DecompressorLZMA(parent, length, cb); dec = new DecompressorLZMA(p, length, cb);
break; break;
case METHOD_LZSS: case METHOD_LZSS:
dec = new DecompressorLZSS(parent, cb); dec = new DecompressorLZSS(p, cb);
break; break;
// todo: METHOD_IMPLODE, METHOD_SHRINK // todo: METHOD_IMPLODE, METHOD_SHRINK
default: default:
return false; return false;
} }
if (method & METHOD_TRANSFEROWNER)
{
dec->SetOwnsReader();
}
dec->Length = (long)length; dec->Length = (long)length;
if (!seekable) if (!seekable)
{ {
@ -666,3 +678,117 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
return false; return false;
} }
} }
//==========================================================================
//
//
//
//==========================================================================
CompressedFileWriter::CompressedFileWriter(FileWriter *targ, bool transfer)
{
target = targ;
zipstream = new z_stream;
compressedSize = 0;
zipstream->next_in = Z_NULL;
zipstream->avail_in = 0;
zipstream->zalloc = Z_NULL;
zipstream->zfree = Z_NULL;
int err = deflateInit2 (zipstream, Z_BEST_COMPRESSION, Z_DEFLATED, -MAX_WBITS, 8, Z_DEFAULT_STRATEGY);
if (err != Z_OK)
{
delete zipstream;
zipstream = nullptr;
return;
}
zipstream->next_out = outbuf;
zipstream->avail_out = sizeof(outbuf);
ownsWriter = transfer;
}
//==========================================================================
//
//
//
//==========================================================================
CompressedFileWriter::CompressedFileWriter(FILE *targ)
: CompressedFileWriter(new FileWriter(targ), true)
{
}
//==========================================================================
//
//
//
//==========================================================================
size_t CompressedFileWriter::Write(const void *buffer, size_t bytes)
{
size_t wrote = 0;
size_t towrite = bytes;
zipstream->next_in = (Bytef *)buffer;
while (bytes > 0)
{
auto chunk = std::min(towrite, (size_t)0x40000000);
zipstream->avail_in = chunk;
buffer = ((char*)buffer) + chunk;
towrite -= chunk;
while (zipstream->avail_in != 0)
{
if (zipstream->avail_out == 0)
{
zipstream->next_out = outbuf;
zipstream->avail_out = 1024;
wrote += 1024;
target->Write(outbuf, 1024);
}
deflate (zipstream, Z_NO_FLUSH);
}
}
compressedSize += wrote;
return bytes;
}
//==========================================================================
//
//
//
//==========================================================================
void CompressedFileWriter::Close()
{
if (!zipstream) return;
// Flush the zlib stream buffer.
for (bool done = false;;)
{
auto len = sizeof(outbuf) - zipstream->avail_out;
if (len != 0)
{
compressedSize += len;
target->Write(outbuf, len);
zipstream->next_out = outbuf;
zipstream->avail_out = sizeof(outbuf);
}
if (done)
{
break;
}
auto err = deflate (zipstream, Z_FINISH);
done = stream.avail_out != 0 || err == Z_STREAM_END;
if (err != Z_STREAM_END && err != Z_OK)
{
break;
}
}
deflateEnd (zipstream);
delete zipstream;
zipstream = nullptr;
}

View file

@ -152,14 +152,36 @@ uint16_t g_nummenusaves;
static menusave_t * g_internalsaves; static menusave_t * g_internalsaves;
static uint16_t g_numinternalsaves; static uint16_t g_numinternalsaves;
static FileReader OpenSavegame(const char *fn)
{
auto file = fopenFileReader(fn, 0);
if (!file.isOpen())
return file;
char buffer[13];
file.Read(buffer, 13);
if (memcmp(buffer, "DEMOLITION_ED", 13))
return FileReader();
FileReader fr;
try
{
fr.OpenDecompressor(file, file.GetLength()-13, METHOD_DEFLATE, false, nullptr);
}
catch(std::runtime_error & err)
{
Printf("%s: %s\n", fn, err.what());
}
return fr;
}
static void ReadSaveGameHeaders_CACHE1D(TArray<FString> &saves) static void ReadSaveGameHeaders_CACHE1D(TArray<FString> &saves)
{ {
savehead_t h; savehead_t h;
for (auto &save : saves) for (FString &save : saves)
{ {
char const * fn = save; auto fil = OpenSavegame(save);
auto fil = fopenFileReader(fn, 0);
if (!fil.isOpen()) if (!fil.isOpen())
continue; continue;
@ -176,7 +198,7 @@ static void ReadSaveGameHeaders_CACHE1D(TArray<FString> &saves)
{ {
if (FURY) if (FURY)
{ {
FStringf extfn("%s.ext", fn); FStringf extfn("%s.ext", save.GetChars());
auto extfil = fopenFileReader(extfn, 0); auto extfil = fopenFileReader(extfn, 0);
if (extfil.isOpen()) if (extfil.isOpen())
{ {
@ -191,7 +213,7 @@ static void ReadSaveGameHeaders_CACHE1D(TArray<FString> &saves)
msv.isAutoSave = h.isAutoSave(); msv.isAutoSave = h.isAutoSave();
strncpy(msv.brief.path, fn, ARRAY_SIZE(msv.brief.path)); strncpy(msv.brief.path, save.GetChars(), ARRAY_SIZE(msv.brief.path));
++g_numinternalsaves; ++g_numinternalsaves;
if (k >= 0 && h.savename[0] != '\0') if (k >= 0 && h.savename[0] != '\0')
@ -293,7 +315,7 @@ void ReadSaveGameHeaders(void)
int32_t G_LoadSaveHeaderNew(char const *fn, savehead_t *saveh) int32_t G_LoadSaveHeaderNew(char const *fn, savehead_t *saveh)
{ {
auto fil = fopenFileReader(fn, 0); auto fil = OpenSavegame(fn);
if (!fil.isOpen()) if (!fil.isOpen())
return -1; return -1;
@ -342,7 +364,7 @@ int32_t G_LoadPlayer(savebrief_t & sv)
int level = -1; int level = -1;
int skill = -1; int skill = -1;
auto fil = fopenFileReader(sv.path, 0); auto fil = OpenSavegame(sv.path);
if (fil.isOpen()) if (fil.isOpen())
{ {
@ -569,7 +591,7 @@ int32_t G_LoadPlayer(savebrief_t & sv)
return 0; return 0;
} }
auto fil = fopenFileReader(sv.path, 0); auto fil = OpenSavegame(sv.path);
if (!fil.isOpen()) if (!fil.isOpen())
return -1; return -1;
@ -762,11 +784,10 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
Bstrcpy(sv.path, fn + (fn.Len() - (ARRAY_SIZE(SaveName) - 1))); Bstrcpy(sv.path, fn + (fn.Len() - (ARRAY_SIZE(SaveName) - 1)));
} }
FileWriter fw(fil);
if (!fil) if (!fil)
{ {
OSD_Printf("G_SavePlayer: failed opening \"%s\" for writing: %s\n", OSD_Printf("G_SavePlayer: failed opening \"%s\" for writing: %s\n",
fn, strerror(errno)); fn.GetChars(), strerror(errno));
ready2send = 1; ready2send = 1;
Net_WaitForServer(); Net_WaitForServer();
@ -776,6 +797,9 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
} }
else else
{ {
fwrite("DEMOLITION_ED", 13, 1, fil);
CompressedFileWriter fw(fil);
sv.isExt = 0; sv.isExt = 0;
// temporary hack // temporary hack
@ -792,7 +816,7 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
if (!g_netServer && ud.multimode < 2) if (!g_netServer && ud.multimode < 2)
{ {
OSD_Printf("Saved: %s\n", fn); OSD_Printf("Saved: %s\n", fn.GetChars());
strcpy(apStrings[QUOTE_RESERVED4], "Game Saved"); strcpy(apStrings[QUOTE_RESERVED4], "Game Saved");
P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps); P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps);
} }

View file

@ -147,14 +147,37 @@ uint16_t g_nummenusaves;
static menusave_t * g_internalsaves; static menusave_t * g_internalsaves;
static uint16_t g_numinternalsaves; static uint16_t g_numinternalsaves;
static FileReader OpenSavegame(const char *fn)
{
auto file = fopenFileReader(fn, 0);
if (!file.isOpen())
return file;
char buffer[13];
file.Read(buffer, 13);
if (memcmp(buffer, "DEMOLITION_RN", 13))
return FileReader();
FileReader fr;
try
{
fr.OpenDecompressor(file, file.GetLength()-13, METHOD_DEFLATE|METHOD_TRANSFEROWNER, false, nullptr);
}
catch(std::runtime_error & err)
{
Printf("%s: %s\n", fn, err.what());
}
return fr;
}
static void ReadSaveGameHeaders_CACHE1D(TArray<FString>& saves) static void ReadSaveGameHeaders_CACHE1D(TArray<FString>& saves)
{ {
savehead_t h; savehead_t h;
for (auto& save : saves) for (FString& save : saves)
{ {
char const* fn = save; char const* fn = save;
auto fil = fopenFileReader(fn, 0); auto fil = OpenSavegame(fn);
if (!fil.isOpen()) if (!fil.isOpen())
continue; continue;
@ -274,7 +297,7 @@ void ReadSaveGameHeaders(void)
int32_t G_LoadSaveHeaderNew(char const *fn, savehead_t *saveh) int32_t G_LoadSaveHeaderNew(char const *fn, savehead_t *saveh)
{ {
auto fil = fopenFileReader(fn, 0); auto fil = OpenSavegame(fn);
if (!fil.isOpen()) if (!fil.isOpen())
return -1; return -1;
@ -316,7 +339,7 @@ static int different_user_map;
// XXX: keyboard input 'blocked' after load fail? (at least ESC?) // XXX: keyboard input 'blocked' after load fail? (at least ESC?)
int32_t G_LoadPlayer(savebrief_t & sv) int32_t G_LoadPlayer(savebrief_t & sv)
{ {
auto fil = fopenFileReader(sv.path, 0); auto fil = OpenSavegame(sv.path);
if (!fil.isOpen()) if (!fil.isOpen())
return -1; return -1;
@ -504,11 +527,10 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
Bstrcpy(sv.path, fn + (fn.Len() - (ARRAY_SIZE(SaveName) - 1))); Bstrcpy(sv.path, fn + (fn.Len() - (ARRAY_SIZE(SaveName) - 1)));
} }
FileWriter fw(fil);
if (!fil) if (!fil)
{ {
OSD_Printf("G_SavePlayer: failed opening \"%s\" for writing: %s\n", OSD_Printf("G_SavePlayer: failed opening \"%s\" for writing: %s\n",
fn, strerror(errno)); fn.GetChars(), strerror(errno));
ready2send = 1; ready2send = 1;
Net_WaitForEverybody(); Net_WaitForEverybody();
@ -518,6 +540,9 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
} }
else else
{ {
fwrite("DEMOLITION_RN", 13, 1, fil);
CompressedFileWriter fw(fil);
// temporary hack // temporary hack
ud.user_map = G_HaveUserMap(); ud.user_map = G_HaveUserMap();
@ -527,19 +552,20 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
fw.Close(); fw.Close();
if (!g_netServer && ud.multimode < 2) if (!g_netServer && ud.multimode < 2)
{ {
Bstrcpy(apStrings[QUOTE_RESERVED4], "Game Saved"); OSD_Printf("Saved: %s\n", fn.GetChars());
P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps); Bstrcpy(apStrings[QUOTE_RESERVED4], "Game Saved");
} P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps);
}
ready2send = 1; ready2send = 1;
Net_WaitForEverybody(); Net_WaitForEverybody();
G_RestoreTimers(); G_RestoreTimers();
ototalclock = totalclock; ototalclock = totalclock;
return 0; return 0;
} }
} }