- implemented savegame compression

Unfortunately necessary because Ion Fury savegames store 120 GB(!!) of data, mostly zeros.

Unlike the old method, this compresses the entire savegame as one block using a ZLib stream so it should be a lot more efficient now.

source/common/filesystem/file_zip.cpp

@@ -68,7 +68,7 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, int L
 		case METHOD_LZMA:
 		{
 			FileReader frz;
-			if (frz.OpenDecompressor(Reader, LumpSize, Method, false, [](const char* err) { I_Error("%s", err); }))
+			if (frz.OpenDecompressor(Reader, LumpSize, Method, false, nullptr))
 			{
 				frz.Read(Cache, LumpSize);
 			}

source/common/utility/files.h

@@ -56,6 +56,7 @@ enum
 	METHOD_PPMD = 98,
 	METHOD_LZSS = 1337,	// not used in Zips - this is for Console Doom compression
 	METHOD_ZLIB = 1338,	// Zlib stream with header, used by compressed nodes.
+	METHOD_TRANSFEROWNER = 0x8000,
 };
 
 class FileReaderInterface
@@ -85,6 +86,11 @@ public:
 	{
 		ErrorCallback = cb;
 	}
+	void SetOwnsReader();
+
+protected:
+	FileReader *File = nullptr;
+	FileReader OwnedFile;
 };
 
 class MemoryReader : public FileReaderInterface
@@ -316,7 +322,7 @@ public:
 	}
 	virtual ~FileWriter()
 	{
-		if (File != NULL) fclose(File);
+		Close();
 	}
 
 	static FileWriter *Open(const char *filename);
@@ -325,7 +331,7 @@ public:
 	virtual long Tell();
 	virtual long Seek(long offset, int mode);
 	size_t Printf(const char *fmt, ...) GCCPRINTF(2,3);
-	void Close()
+	virtual void Close()
 	{
 		if (File != NULL) fclose(File);
 		File = nullptr;
@@ -351,4 +357,23 @@ public:
 	TArray<unsigned char>&& TakeBuffer() { return std::move(mBuffer); }
 };
 
+class CompressedFileWriter : public FileWriter
+{
+	FileWriter *target;
+	struct z_stream_s *zipstream;
+	uint8_t outbuf[1024];
+	size_t compressedSize;
+	bool ownsWriter;
+	
+	size_t WriteBlock(const void *buffer, size_t bytes);
+
+public:
+	CompressedFileWriter(FileWriter *wr, bool transfer = false);
+	CompressedFileWriter(FILE *wr);
+	~CompressedFileWriter() { Close(); }
+	virtual size_t Write(const void *buffer, size_t len) override;
+	virtual void Close() override;
+
+};
+
 #endif

source/common/utility/files_decompress.cpp

@@ -63,7 +63,7 @@ void DecompressorBase::DecompressionError(const char *error, ...) const
 	va_end(argptr);
 
 	if (ErrorCallback != nullptr) ErrorCallback(errortext);
-	else std::terminate();
+	else throw std::runtime_error(errortext);
 }
 
 long DecompressorBase::Tell () const
@@ -82,6 +82,12 @@ char *DecompressorBase::Gets(char *strbuf, int len)
 	return nullptr;
 }
 
+void DecompressorBase::SetOwnsReader()
+{
+	OwnedFile = std::move(*File);
+	File = &OwnedFile;
+}
+
 //
 // M_ZlibError
 //
@@ -125,17 +131,17 @@ class DecompressorZ : public DecompressorBase
 {
 	enum { BUFF_SIZE = 4096 };
 
-	FileReader &File;
 	bool SawEOF;
 	z_stream Stream;
 	uint8_t InBuff[BUFF_SIZE];
 	
 public:
-	DecompressorZ (FileReader &file, bool zip, const std::function<void(const char*)>& cb)
+	DecompressorZ (FileReader *file, bool zip, const std::function<void(const char*)>& cb)
-	: File(file), SawEOF(false)
+	: SawEOF(false)
 	{
 		int err;
 
+		File = file;
 		SetErrorCallback(cb);
 		FillBuffer ();
 
@@ -187,7 +193,7 @@ public:
 
 	void FillBuffer ()
 	{
-		auto numread = File.Read (InBuff, BUFF_SIZE);
+		auto numread = File->Read (InBuff, BUFF_SIZE);
 
 		if (numread < BUFF_SIZE)
 		{
@@ -216,17 +222,17 @@ class DecompressorBZ2 : public DecompressorBase
 {
 	enum { BUFF_SIZE = 4096 };
 
-	FileReader &File;
 	bool SawEOF;
 	bz_stream Stream;
 	uint8_t InBuff[BUFF_SIZE];
 	
 public:
-	DecompressorBZ2 (FileReader &file, const std::function<void(const char*)>& cb)
+	DecompressorBZ2 (FileReader *file, const std::function<void(const char*)>& cb)
-	: File(file), SawEOF(false)
+	: SawEOF(false)
 	{
 		int err;
 
+		File = file;
 		SetErrorCallback(cb);
 		stupidGlobal = this;
 		FillBuffer ();
@@ -281,7 +287,7 @@ public:
 
 	void FillBuffer ()
 	{
-		auto numread = File.Read(InBuff, BUFF_SIZE);
+		auto numread = File->Read(InBuff, BUFF_SIZE);
 
 		if (numread < BUFF_SIZE)
 		{
@@ -325,7 +331,6 @@ class DecompressorLZMA : public DecompressorBase
 {
 	enum { BUFF_SIZE = 4096 };
 
-	FileReader &File;
 	bool SawEOF;
 	CLzmaDec Stream;
 	size_t Size;
@@ -335,18 +340,19 @@ class DecompressorLZMA : public DecompressorBase
 
 public:
 
-	DecompressorLZMA (FileReader &file, size_t uncompressed_size, const std::function<void(const char*)>& cb)
+	DecompressorLZMA (FileReader *file, size_t uncompressed_size, const std::function<void(const char*)>& cb)
-	: File(file), SawEOF(false)
+	: SawEOF(false)
 	{
 		uint8_t header[4 + LZMA_PROPS_SIZE];
 		int err;
+		File = file;
 		SetErrorCallback(cb);
 
 		Size = uncompressed_size;
 		OutProcessed = 0;
 
 		// Read zip LZMA properties header
-		if (File.Read(header, sizeof(header)) < (long)sizeof(header))
+		if (File->Read(header, sizeof(header)) < (long)sizeof(header))
 		{
 			DecompressionError("DecompressorLZMA: File too short\n");
 		}
@@ -423,7 +429,7 @@ public:
 
 	void FillBuffer ()
 	{
-		auto numread = File.Read(InBuff, BUFF_SIZE);
+		auto numread = File->Read(InBuff, BUFF_SIZE);
 
 		if (numread < BUFF_SIZE)
 		{
@@ -445,7 +451,6 @@ class DecompressorLZSS : public DecompressorBase
 {
 	enum { BUFF_SIZE = 4096, WINDOW_SIZE = 4096, INTERNAL_BUFFER_SIZE = 128 };
 
-	FileReader &File;
 	bool SawEOF;
 	uint8_t InBuff[BUFF_SIZE];
 
@@ -476,7 +481,7 @@ class DecompressorLZSS : public DecompressorBase
 		if(Stream.AvailIn)
 			memmove(InBuff, Stream.In, Stream.AvailIn);
 
-		auto numread = File.Read(InBuff+Stream.AvailIn, BUFF_SIZE-Stream.AvailIn);
+		auto numread = File->Read(InBuff+Stream.AvailIn, BUFF_SIZE-Stream.AvailIn);
 
 		if (numread < BUFF_SIZE)
 		{
@@ -563,8 +568,9 @@ class DecompressorLZSS : public DecompressorBase
 	}
 
 public:
-	DecompressorLZSS(FileReader &file, const std::function<void(const char*)>& cb) : File(file), SawEOF(false)
+	DecompressorLZSS(FileReader *file, const std::function<void(const char*)>& cb) : File(file), SawEOF(false)
 	{
+		File = file;
 		SetErrorCallback(cb);
 		Stream.State = STREAM_EMPTY;
 		Stream.WindowData = Stream.InternalBuffer = Stream.Window+WINDOW_SIZE;
@@ -629,29 +635,35 @@ public:
 bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, const std::function<void(const char*)>& cb)
 {
 	DecompressorBase *dec = nullptr;
-	switch (method)
+	FileReader *p = &parent;
+	switch (method & ~METHOD_TRANSFEROWNER)
 	{
 		case METHOD_DEFLATE:
 		case METHOD_ZLIB:
-			dec = new DecompressorZ(parent, method == METHOD_DEFLATE, cb);
+			dec = new DecompressorZ(p, method == METHOD_DEFLATE, cb);
 			break;
 
 		case METHOD_BZIP2:
-			dec = new DecompressorBZ2(parent, cb);
+			dec = new DecompressorBZ2(p, cb);
 			break;
 
 		case METHOD_LZMA:
-			dec = new DecompressorLZMA(parent, length, cb);
+			dec = new DecompressorLZMA(p, length, cb);
 			break;
 
 		case METHOD_LZSS:
-			dec = new DecompressorLZSS(parent, cb);
+			dec = new DecompressorLZSS(p, cb);
 			break;
 			
 		// todo: METHOD_IMPLODE, METHOD_SHRINK
 		default:
 			return false;
 	}
+	if (method & METHOD_TRANSFEROWNER)
+	{
+		dec->SetOwnsReader();
+	}
+
 	dec->Length = (long)length;
 	if (!seekable)
 	{
@@ -666,3 +678,117 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 		return false;
 	}
 }
+
+
+
+//==========================================================================
+//
+//
+//
+//==========================================================================
+						
+CompressedFileWriter::CompressedFileWriter(FileWriter *targ, bool transfer)
+{
+	target = targ;
+	zipstream = new z_stream;
+
+	compressedSize = 0;
+	zipstream->next_in = Z_NULL;
+	zipstream->avail_in = 0;
+	zipstream->zalloc = Z_NULL;
+	zipstream->zfree = Z_NULL;
+	int err = deflateInit2 (zipstream, Z_BEST_COMPRESSION, Z_DEFLATED, -MAX_WBITS, 8, Z_DEFAULT_STRATEGY);
+	if (err != Z_OK)
+	{
+		delete zipstream;
+		zipstream = nullptr;
+		return;
+	}
+	zipstream->next_out = outbuf;
+	zipstream->avail_out = sizeof(outbuf);
+	ownsWriter = transfer;
+}
+
+//==========================================================================
+//
+//
+//
+//==========================================================================
+						
+CompressedFileWriter::CompressedFileWriter(FILE *targ)
+ : CompressedFileWriter(new FileWriter(targ), true)
+{
+}
+
+//==========================================================================
+//
+//
+//
+//==========================================================================
+
+size_t CompressedFileWriter::Write(const void *buffer, size_t bytes)
+{
+	size_t wrote = 0;
+	size_t towrite = bytes;
+	
+	zipstream->next_in = (Bytef *)buffer;
+	while (bytes > 0)
+	{
+		auto chunk = std::min(towrite, (size_t)0x40000000);
+		zipstream->avail_in = chunk;
+		buffer = ((char*)buffer) + chunk;
+		towrite -= chunk;
+		
+		while (zipstream->avail_in != 0)
+		{
+			if (zipstream->avail_out == 0)
+			{
+				zipstream->next_out = outbuf;
+				zipstream->avail_out = 1024;
+				wrote += 1024;
+				target->Write(outbuf, 1024);
+			}
+			deflate (zipstream, Z_NO_FLUSH);
+		}
+	}
+	compressedSize += wrote;
+	return bytes;
+}
+
+//==========================================================================
+//
+//
+//
+//==========================================================================
+
+void CompressedFileWriter::Close()
+{
+	if (!zipstream) return;
+	// Flush the zlib stream buffer.
+	
+	for (bool done = false;;)
+	{
+		auto len = sizeof(outbuf) - zipstream->avail_out;
+		if (len != 0)
+		{
+			compressedSize += len;
+			
+			target->Write(outbuf, len);
+			zipstream->next_out = outbuf;
+			zipstream->avail_out = sizeof(outbuf);
+		}
+		if (done)
+		{
+			break;
+		}
+		auto err = deflate (zipstream, Z_FINISH);
+		done = stream.avail_out != 0 || err == Z_STREAM_END;
+		if (err != Z_STREAM_END && err != Z_OK)
+		{
+			break;
+		}
+	}
+	deflateEnd (zipstream);
+	delete zipstream;
+	zipstream = nullptr;
+}

source/duke3d/src/savegame.cpp

@@ -152,14 +152,36 @@ uint16_t g_nummenusaves;
 static menusave_t * g_internalsaves;
 static uint16_t g_numinternalsaves;
 
+static FileReader OpenSavegame(const char *fn)
+{
+	auto file = fopenFileReader(fn, 0);
+	if (!file.isOpen())
+		return file;
+
+	char buffer[13];
+	file.Read(buffer, 13);
+	if (memcmp(buffer, "DEMOLITION_ED", 13))
+		return FileReader();
+	
+	FileReader fr;
+	try
+	{
+		fr.OpenDecompressor(file, file.GetLength()-13, METHOD_DEFLATE, false, nullptr);
+	}
+	catch(std::runtime_error & err)
+	{
+		Printf("%s: %s\n", fn, err.what());
+	}
+	return fr;
+}
+
 static void ReadSaveGameHeaders_CACHE1D(TArray<FString> &saves)
 {
     savehead_t h;
 
-	for (auto &save : saves)
+	for (FString &save : saves)
     {
-        char const * fn = save;
+		auto fil = OpenSavegame(save);
-        auto fil = fopenFileReader(fn, 0);
         if (!fil.isOpen())
             continue;
 
@@ -176,7 +198,7 @@ static void ReadSaveGameHeaders_CACHE1D(TArray<FString> &saves)
             {
                 if (FURY)
                 {
-					FStringf extfn("%s.ext", fn);
+					FStringf extfn("%s.ext", save.GetChars());
 					auto extfil = fopenFileReader(extfn, 0);
 					if (extfil.isOpen())
 					{
@@ -191,7 +213,7 @@ static void ReadSaveGameHeaders_CACHE1D(TArray<FString> &saves)
 
         msv.isAutoSave = h.isAutoSave();
 
-        strncpy(msv.brief.path, fn, ARRAY_SIZE(msv.brief.path));
+        strncpy(msv.brief.path, save.GetChars(), ARRAY_SIZE(msv.brief.path));
         ++g_numinternalsaves;
 
         if (k >= 0 && h.savename[0] != '\0')
@@ -293,7 +315,7 @@ void ReadSaveGameHeaders(void)
 
 int32_t G_LoadSaveHeaderNew(char const *fn, savehead_t *saveh)
 {
-    auto fil = fopenFileReader(fn, 0);
+    auto fil = OpenSavegame(fn);
     if (!fil.isOpen())
         return -1;
 
@@ -342,7 +364,7 @@ int32_t G_LoadPlayer(savebrief_t & sv)
         int level = -1;
         int skill = -1;
 
-		auto fil = fopenFileReader(sv.path, 0);
+		auto fil = OpenSavegame(sv.path);
 
         if (fil.isOpen())
         {
@@ -569,7 +591,7 @@ int32_t G_LoadPlayer(savebrief_t & sv)
         return 0;
     }
 
-    auto fil = fopenFileReader(sv.path, 0);
+    auto fil = OpenSavegame(sv.path);
 
     if (!fil.isOpen())
         return -1;
@@ -762,11 +784,10 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
 		Bstrcpy(sv.path, fn + (fn.Len() - (ARRAY_SIZE(SaveName) - 1)));
 	}
 
-	FileWriter fw(fil);
 	if (!fil)
 	{
 		OSD_Printf("G_SavePlayer: failed opening \"%s\" for writing: %s\n",
-			fn, strerror(errno));
+			fn.GetChars(), strerror(errno));
 		ready2send = 1;
 		Net_WaitForServer();
 
@@ -776,6 +797,9 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
 	}
 	else
 	{
+		fwrite("DEMOLITION_ED", 13, 1, fil);
+		CompressedFileWriter fw(fil);
+
 		sv.isExt = 0;
 
 		// temporary hack
@@ -792,7 +816,7 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
 
 		if (!g_netServer && ud.multimode < 2)
 		{
-			OSD_Printf("Saved: %s\n", fn);
+			OSD_Printf("Saved: %s\n", fn.GetChars());
 			strcpy(apStrings[QUOTE_RESERVED4], "Game Saved");
 			P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps);
 		}

source/rr/src/savegame.cpp

@@ -147,14 +147,37 @@ uint16_t g_nummenusaves;
 static menusave_t * g_internalsaves;
 static uint16_t g_numinternalsaves;
 
+static FileReader OpenSavegame(const char *fn)
+{
+	auto file = fopenFileReader(fn, 0);
+	if (!file.isOpen())
+		return file;
+
+	char buffer[13];
+	file.Read(buffer, 13);
+	if (memcmp(buffer, "DEMOLITION_RN", 13))
+		return FileReader();
+	
+	FileReader fr;
+	try
+	{
+		fr.OpenDecompressor(file, file.GetLength()-13, METHOD_DEFLATE|METHOD_TRANSFEROWNER, false, nullptr);
+	}
+	catch(std::runtime_error & err)
+	{
+		Printf("%s: %s\n", fn, err.what());
+	}
+	return fr;
+}
+
 static void ReadSaveGameHeaders_CACHE1D(TArray<FString>& saves)
 {
     savehead_t h;
 
-	for (auto& save : saves)
+	for (FString& save : saves)
 	{
 		char const* fn = save;
-		auto fil = fopenFileReader(fn, 0);
+		auto fil = OpenSavegame(fn);
         if (!fil.isOpen())
             continue;
 
@@ -274,7 +297,7 @@ void ReadSaveGameHeaders(void)
 
 int32_t G_LoadSaveHeaderNew(char const *fn, savehead_t *saveh)
 {
-    auto fil = fopenFileReader(fn, 0);
+    auto fil = OpenSavegame(fn);
     if (!fil.isOpen())
         return -1;
 
@@ -316,7 +339,7 @@ static int different_user_map;
 // XXX: keyboard input 'blocked' after load fail? (at least ESC?)
 int32_t G_LoadPlayer(savebrief_t & sv)
 {
-    auto fil = fopenFileReader(sv.path, 0);
+    auto fil = OpenSavegame(sv.path);
 
     if (!fil.isOpen())
         return -1;
@@ -504,11 +527,10 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
 		Bstrcpy(sv.path, fn + (fn.Len() - (ARRAY_SIZE(SaveName) - 1)));
 	}
 
-	FileWriter fw(fil);
 	if (!fil)
 	{
 		OSD_Printf("G_SavePlayer: failed opening \"%s\" for writing: %s\n",
-			fn, strerror(errno));
+			fn.GetChars(), strerror(errno));
 		ready2send = 1;
 		Net_WaitForEverybody();
 
@@ -518,6 +540,9 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
 	}
 	else
 	{
+		fwrite("DEMOLITION_RN", 13, 1, fil);
+		CompressedFileWriter fw(fil);
+
 		// temporary hack
 		ud.user_map = G_HaveUserMap();
 
@@ -529,6 +554,7 @@ int32_t G_SavePlayer(savebrief_t & sv, bool isAutoSave)
 
 		if (!g_netServer && ud.multimode < 2)
 		{
+			OSD_Printf("Saved: %s\n", fn.GetChars());
 			Bstrcpy(apStrings[QUOTE_RESERVED4], "Game Saved");
 			P_DoQuote(QUOTE_RESERVED4, g_player[myconnectindex].ps);
 		}