From 4fc56203c27d82860ff93c93d0e9c06cc3ac8c21 Mon Sep 17 00:00:00 2001 From: Christoph Oelckers Date: Fri, 8 Nov 2019 01:36:32 +0100 Subject: [PATCH] - implemented savegame compression Unfortunately necessary because Ion Fury savegames store 120 GB(!!) of data, mostly zeros. Unlike the old method, this compresses the entire savegame as one block using a ZLib stream so it should be a lot more efficient now. --- source/common/filesystem/file_zip.cpp | 2 +- source/common/utility/files.h | 29 +++- source/common/utility/files_decompress.cpp | 170 ++++++++++++++++++--- source/duke3d/src/savegame.cpp | 46 ++++-- source/rr/src/savegame.cpp | 64 +++++--- 5 files changed, 256 insertions(+), 55 deletions(-) diff --git a/source/common/filesystem/file_zip.cpp b/source/common/filesystem/file_zip.cpp index 7790d78b6..579391147 100644 --- a/source/common/filesystem/file_zip.cpp +++ b/source/common/filesystem/file_zip.cpp @@ -68,7 +68,7 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, int L case METHOD_LZMA: { FileReader frz; - if (frz.OpenDecompressor(Reader, LumpSize, Method, false, [](const char* err) { I_Error("%s", err); })) + if (frz.OpenDecompressor(Reader, LumpSize, Method, false, nullptr)) { frz.Read(Cache, LumpSize); } diff --git a/source/common/utility/files.h b/source/common/utility/files.h index 8cf17bfd8..d9eb00b1c 100644 --- a/source/common/utility/files.h +++ b/source/common/utility/files.h @@ -56,6 +56,7 @@ enum METHOD_PPMD = 98, METHOD_LZSS = 1337, // not used in Zips - this is for Console Doom compression METHOD_ZLIB = 1338, // Zlib stream with header, used by compressed nodes. + METHOD_TRANSFEROWNER = 0x8000, }; class FileReaderInterface @@ -85,6 +86,11 @@ public: { ErrorCallback = cb; } + void SetOwnsReader(); + +protected: + FileReader *File = nullptr; + FileReader OwnedFile; }; class MemoryReader : public FileReaderInterface @@ -316,7 +322,7 @@ public: } virtual ~FileWriter() { - if (File != NULL) fclose(File); + Close(); } static FileWriter *Open(const char *filename); @@ -325,7 +331,7 @@ public: virtual long Tell(); virtual long Seek(long offset, int mode); size_t Printf(const char *fmt, ...) GCCPRINTF(2,3); - void Close() + virtual void Close() { if (File != NULL) fclose(File); File = nullptr; @@ -351,4 +357,23 @@ public: TArray&& TakeBuffer() { return std::move(mBuffer); } }; +class CompressedFileWriter : public FileWriter +{ + FileWriter *target; + struct z_stream_s *zipstream; + uint8_t outbuf[1024]; + size_t compressedSize; + bool ownsWriter; + + size_t WriteBlock(const void *buffer, size_t bytes); + +public: + CompressedFileWriter(FileWriter *wr, bool transfer = false); + CompressedFileWriter(FILE *wr); + ~CompressedFileWriter() { Close(); } + virtual size_t Write(const void *buffer, size_t len) override; + virtual void Close() override; + +}; + #endif diff --git a/source/common/utility/files_decompress.cpp b/source/common/utility/files_decompress.cpp index 4770c1440..7136620e6 100644 --- a/source/common/utility/files_decompress.cpp +++ b/source/common/utility/files_decompress.cpp @@ -63,7 +63,7 @@ void DecompressorBase::DecompressionError(const char *error, ...) const va_end(argptr); if (ErrorCallback != nullptr) ErrorCallback(errortext); - else std::terminate(); + else throw std::runtime_error(errortext); } long DecompressorBase::Tell () const @@ -82,6 +82,12 @@ char *DecompressorBase::Gets(char *strbuf, int len) return nullptr; } +void DecompressorBase::SetOwnsReader() +{ + OwnedFile = std::move(*File); + File = &OwnedFile; +} + // // M_ZlibError // @@ -125,17 +131,17 @@ class DecompressorZ : public DecompressorBase { enum { BUFF_SIZE = 4096 }; - FileReader &File; bool SawEOF; z_stream Stream; uint8_t InBuff[BUFF_SIZE]; public: - DecompressorZ (FileReader &file, bool zip, const std::function& cb) - : File(file), SawEOF(false) + DecompressorZ (FileReader *file, bool zip, const std::function& cb) + : SawEOF(false) { int err; + File = file; SetErrorCallback(cb); FillBuffer (); @@ -187,7 +193,7 @@ public: void FillBuffer () { - auto numread = File.Read (InBuff, BUFF_SIZE); + auto numread = File->Read (InBuff, BUFF_SIZE); if (numread < BUFF_SIZE) { @@ -216,17 +222,17 @@ class DecompressorBZ2 : public DecompressorBase { enum { BUFF_SIZE = 4096 }; - FileReader &File; bool SawEOF; bz_stream Stream; uint8_t InBuff[BUFF_SIZE]; public: - DecompressorBZ2 (FileReader &file, const std::function& cb) - : File(file), SawEOF(false) + DecompressorBZ2 (FileReader *file, const std::function& cb) + : SawEOF(false) { int err; + File = file; SetErrorCallback(cb); stupidGlobal = this; FillBuffer (); @@ -281,7 +287,7 @@ public: void FillBuffer () { - auto numread = File.Read(InBuff, BUFF_SIZE); + auto numread = File->Read(InBuff, BUFF_SIZE); if (numread < BUFF_SIZE) { @@ -325,7 +331,6 @@ class DecompressorLZMA : public DecompressorBase { enum { BUFF_SIZE = 4096 }; - FileReader &File; bool SawEOF; CLzmaDec Stream; size_t Size; @@ -335,18 +340,19 @@ class DecompressorLZMA : public DecompressorBase public: - DecompressorLZMA (FileReader &file, size_t uncompressed_size, const std::function& cb) - : File(file), SawEOF(false) + DecompressorLZMA (FileReader *file, size_t uncompressed_size, const std::function& cb) + : SawEOF(false) { uint8_t header[4 + LZMA_PROPS_SIZE]; int err; + File = file; SetErrorCallback(cb); Size = uncompressed_size; OutProcessed = 0; // Read zip LZMA properties header - if (File.Read(header, sizeof(header)) < (long)sizeof(header)) + if (File->Read(header, sizeof(header)) < (long)sizeof(header)) { DecompressionError("DecompressorLZMA: File too short\n"); } @@ -423,7 +429,7 @@ public: void FillBuffer () { - auto numread = File.Read(InBuff, BUFF_SIZE); + auto numread = File->Read(InBuff, BUFF_SIZE); if (numread < BUFF_SIZE) { @@ -445,7 +451,6 @@ class DecompressorLZSS : public DecompressorBase { enum { BUFF_SIZE = 4096, WINDOW_SIZE = 4096, INTERNAL_BUFFER_SIZE = 128 }; - FileReader &File; bool SawEOF; uint8_t InBuff[BUFF_SIZE]; @@ -476,7 +481,7 @@ class DecompressorLZSS : public DecompressorBase if(Stream.AvailIn) memmove(InBuff, Stream.In, Stream.AvailIn); - auto numread = File.Read(InBuff+Stream.AvailIn, BUFF_SIZE-Stream.AvailIn); + auto numread = File->Read(InBuff+Stream.AvailIn, BUFF_SIZE-Stream.AvailIn); if (numread < BUFF_SIZE) { @@ -563,8 +568,9 @@ class DecompressorLZSS : public DecompressorBase } public: - DecompressorLZSS(FileReader &file, const std::function& cb) : File(file), SawEOF(false) + DecompressorLZSS(FileReader *file, const std::function& cb) : File(file), SawEOF(false) { + File = file; SetErrorCallback(cb); Stream.State = STREAM_EMPTY; Stream.WindowData = Stream.InternalBuffer = Stream.Window+WINDOW_SIZE; @@ -629,29 +635,35 @@ public: bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, const std::function& cb) { DecompressorBase *dec = nullptr; - switch (method) + FileReader *p = &parent; + switch (method & ~METHOD_TRANSFEROWNER) { case METHOD_DEFLATE: case METHOD_ZLIB: - dec = new DecompressorZ(parent, method == METHOD_DEFLATE, cb); + dec = new DecompressorZ(p, method == METHOD_DEFLATE, cb); break; case METHOD_BZIP2: - dec = new DecompressorBZ2(parent, cb); + dec = new DecompressorBZ2(p, cb); break; case METHOD_LZMA: - dec = new DecompressorLZMA(parent, length, cb); + dec = new DecompressorLZMA(p, length, cb); break; case METHOD_LZSS: - dec = new DecompressorLZSS(parent, cb); + dec = new DecompressorLZSS(p, cb); break; // todo: METHOD_IMPLODE, METHOD_SHRINK default: return false; } + if (method & METHOD_TRANSFEROWNER) + { + dec->SetOwnsReader(); + } + dec->Length = (long)length; if (!seekable) { @@ -666,3 +678,117 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b return false; } } + + + +//========================================================================== +// +// +// +//========================================================================== + +CompressedFileWriter::CompressedFileWriter(FileWriter *targ, bool transfer) +{ + target = targ; + zipstream = new z_stream; + + compressedSize = 0; + zipstream->next_in = Z_NULL; + zipstream->avail_in = 0; + zipstream->zalloc = Z_NULL; + zipstream->zfree = Z_NULL; + int err = deflateInit2 (zipstream, Z_BEST_COMPRESSION, Z_DEFLATED, -MAX_WBITS, 8, Z_DEFAULT_STRATEGY); + if (err != Z_OK) + { + delete zipstream; + zipstream = nullptr; + return; + } + zipstream->next_out = outbuf; + zipstream->avail_out = sizeof(outbuf); + ownsWriter = transfer; +} + +//========================================================================== +// +// +// +//========================================================================== + +CompressedFileWriter::CompressedFileWriter(FILE *targ) + : CompressedFileWriter(new FileWriter(targ), true) +{ +} + +//========================================================================== +// +// +// +//========================================================================== + +size_t CompressedFileWriter::Write(const void *buffer, size_t bytes) +{ + size_t wrote = 0; + size_t towrite = bytes; + + zipstream->next_in = (Bytef *)buffer; + while (bytes > 0) + { + auto chunk = std::min(towrite, (size_t)0x40000000); + zipstream->avail_in = chunk; + buffer = ((char*)buffer) + chunk; + towrite -= chunk; + + while (zipstream->avail_in != 0) + { + if (zipstream->avail_out == 0) + { + zipstream->next_out = outbuf; + zipstream->avail_out = 1024; + wrote += 1024; + target->Write(outbuf, 1024); + } + deflate (zipstream, Z_NO_FLUSH); + } + } + compressedSize += wrote; + return bytes; +} + +//========================================================================== +// +// +// +//========================================================================== + +void CompressedFileWriter::Close() +{ + if (!zipstream) return; + // Flush the zlib stream buffer. + + for (bool done = false;;) + { + auto len = sizeof(outbuf) - zipstream->avail_out; + if (len != 0) + { + compressedSize += len; + + target->Write(outbuf, len); + zipstream->next_out = outbuf; + zipstream->avail_out = sizeof(outbuf); + } + if (done) + { + break; + } + auto err = deflate (zipstream, Z_FINISH); + done = stream.avail_out != 0 || err == Z_STREAM_END; + if (err != Z_STREAM_END && err != Z_OK) + { + break; + } + } + deflateEnd (zipstream); + delete zipstream; + zipstream = nullptr; +} diff --git a/source/duke3d/src/savegame.cpp b/source/duke3d/src/savegame.cpp index a4b9ab68c..49d477ac7 100644 --- a/source/duke3d/src/savegame.cpp +++ b/source/duke3d/src/savegame.cpp @@ -152,14 +152,36 @@ uint16_t g_nummenusaves; static menusave_t * g_internalsaves; static uint16_t g_numinternalsaves; +static FileReader OpenSavegame(const char *fn) +{ + auto file = fopenFileReader(fn, 0); + if (!file.isOpen()) + return file; + + char buffer[13]; + file.Read(buffer, 13); + if (memcmp(buffer, "DEMOLITION_ED", 13)) + return FileReader(); + + FileReader fr; + try + { + fr.OpenDecompressor(file, file.GetLength()-13, METHOD_DEFLATE, false, nullptr); + } + catch(std::runtime_error & err) + { + Printf("%s: %s\n", fn, err.what()); + } + return fr; +} + static void ReadSaveGameHeaders_CACHE1D(TArray &saves) { savehead_t h; - for (auto &save : saves) + for (FString &save : saves) { - char const * fn = save; - auto fil = fopenFileReader(fn, 0); + auto fil = OpenSavegame(save); if (!fil.isOpen()) continue; @@ -176,7 +198,7 @@ static void ReadSaveGameHeaders_CACHE1D(TArray &saves) { if (FURY) { - FStringf extfn("%s.ext", fn); + FStringf extfn("%s.ext", save.GetChars()); auto extfil = fopenFileReader(extfn, 0); if (extfil.isOpen()) { @@ -191,7 +213,7 @@ static void ReadSaveGameHeaders_CACHE1D(TArray &saves) msv.isAutoSave = h.isAutoSave(); - strncpy(msv.brief.path, fn, ARRAY_SIZE(msv.brief.path)); + strncpy(msv.brief.path, save.GetChars(), ARRAY_SIZE(msv.brief.path)); ++g_numinternalsaves; if (k >= 0 && h.savename[0] != '\0') @@ -293,7 +315,7 @@ void ReadSaveGameHeaders(void) int32_t G_LoadSaveHeaderNew(char const *fn, savehead_t *saveh) { - auto fil = fopenFileReader(fn, 0); + auto fil = OpenSavegame(fn); if (!fil.isOpen()) return -1; @@ -342,7 +364,7 @@ int32_t G_LoadPlayer(savebrief_t & sv) int level = -1; int skill = -1; - auto fil = fopenFileReader(sv.path, 0); + auto fil = OpenSavegame(sv.path); if (fil.isOpen()) { @@ -569,7 +591,7 @@ int32_t G_LoadPlayer(savebrief_t & sv) return 0; } - auto fil = fopenFileReader(sv.path, 0); + auto fil = OpenSavegame(sv.path); if (!fil.isOpen()) return -1; @@ -762,11 +784,10 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave) Bstrcpy(sv.path, fn + (fn.Len() - (ARRAY_SIZE(SaveName) - 1))); } - FileWriter fw(fil); if (!fil) { OSD_Printf("G_SavePlayer: failed opening \"%s\" for writing: %s\n", - fn, strerror(errno)); + fn.GetChars(), strerror(errno)); ready2send = 1; Net_WaitForServer(); @@ -776,6 +797,9 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave) } else { + fwrite("DEMOLITION_ED", 13, 1, fil); + CompressedFileWriter fw(fil); + sv.isExt = 0; // temporary hack @@ -792,7 +816,7 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave) if (!g_netServer && ud.multimode < 2) { - OSD_Printf("Saved: %s\n", fn); + OSD_Printf("Saved: %s\n", fn.GetChars()); strcpy(apStrings[QUOTE_RESERVED4], "Game Saved"); P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps); } diff --git a/source/rr/src/savegame.cpp b/source/rr/src/savegame.cpp index 52f7f3961..45ec8562b 100644 --- a/source/rr/src/savegame.cpp +++ b/source/rr/src/savegame.cpp @@ -147,14 +147,37 @@ uint16_t g_nummenusaves; static menusave_t * g_internalsaves; static uint16_t g_numinternalsaves; +static FileReader OpenSavegame(const char *fn) +{ + auto file = fopenFileReader(fn, 0); + if (!file.isOpen()) + return file; + + char buffer[13]; + file.Read(buffer, 13); + if (memcmp(buffer, "DEMOLITION_RN", 13)) + return FileReader(); + + FileReader fr; + try + { + fr.OpenDecompressor(file, file.GetLength()-13, METHOD_DEFLATE|METHOD_TRANSFEROWNER, false, nullptr); + } + catch(std::runtime_error & err) + { + Printf("%s: %s\n", fn, err.what()); + } + return fr; +} + static void ReadSaveGameHeaders_CACHE1D(TArray& saves) { savehead_t h; - for (auto& save : saves) + for (FString& save : saves) { char const* fn = save; - auto fil = fopenFileReader(fn, 0); + auto fil = OpenSavegame(fn); if (!fil.isOpen()) continue; @@ -274,7 +297,7 @@ void ReadSaveGameHeaders(void) int32_t G_LoadSaveHeaderNew(char const *fn, savehead_t *saveh) { - auto fil = fopenFileReader(fn, 0); + auto fil = OpenSavegame(fn); if (!fil.isOpen()) return -1; @@ -316,7 +339,7 @@ static int different_user_map; // XXX: keyboard input 'blocked' after load fail? (at least ESC?) int32_t G_LoadPlayer(savebrief_t & sv) { - auto fil = fopenFileReader(sv.path, 0); + auto fil = OpenSavegame(sv.path); if (!fil.isOpen()) return -1; @@ -504,11 +527,10 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave) Bstrcpy(sv.path, fn + (fn.Len() - (ARRAY_SIZE(SaveName) - 1))); } - FileWriter fw(fil); if (!fil) { OSD_Printf("G_SavePlayer: failed opening \"%s\" for writing: %s\n", - fn, strerror(errno)); + fn.GetChars(), strerror(errno)); ready2send = 1; Net_WaitForEverybody(); @@ -518,6 +540,9 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave) } else { + fwrite("DEMOLITION_RN", 13, 1, fil); + CompressedFileWriter fw(fil); + // temporary hack ud.user_map = G_HaveUserMap(); @@ -527,19 +552,20 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave) fw.Close(); - if (!g_netServer && ud.multimode < 2) - { - Bstrcpy(apStrings[QUOTE_RESERVED4], "Game Saved"); - P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps); - } - - ready2send = 1; - Net_WaitForEverybody(); - - G_RestoreTimers(); - ototalclock = totalclock; - - return 0; + if (!g_netServer && ud.multimode < 2) + { + OSD_Printf("Saved: %s\n", fn.GetChars()); + Bstrcpy(apStrings[QUOTE_RESERVED4], "Game Saved"); + P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps); + } + + ready2send = 1; + Net_WaitForEverybody(); + + G_RestoreTimers(); + ototalclock = totalclock; + + return 0; } }