preparations for getting rid of FZipLump

* allow ancient compression algorithms to be handled by OpenDecompressor.
* move FCompressedBuffer to fs_files.h
* use a mutex lock for 7z access because it cannot be made thread save otherwise.

src/common/filesystem/include/fs_files.h

@@ -175,6 +175,29 @@ public:
 
 };
 
+// This holds a compresed Zip entry with all needed info to decompress it.
+struct FCompressedBuffer
+{
+	size_t mSize;
+	size_t mCompressedSize;
+	int mMethod;
+	unsigned mCRC32;
+	char* mBuffer;
+	const char* filename;
+
+	bool Decompress(char* destbuffer);
+	void Clean()
+	{
+		mSize = mCompressedSize = 0;
+		if (mBuffer != nullptr)
+		{
+			delete[] mBuffer;
+			mBuffer = nullptr;
+		}
+	}
+};
+
+
 class FileReaderInterface
 {
 public:

src/common/filesystem/include/resourcefile.h

@@ -92,28 +92,6 @@ enum ELumpFlags
 	
 };
 
-// This holds a compresed Zip entry with all needed info to decompress it.
-struct FCompressedBuffer
-{
-	size_t mSize;
-	size_t mCompressedSize;
-	int mMethod;
-	unsigned mCRC32;
-	char *mBuffer;
-	const char* filename;
-
-	bool Decompress(char *destbuffer);
-	void Clean()
-	{
-		mSize = mCompressedSize = 0;
-		if (mBuffer != nullptr)
-		{
-			delete[] mBuffer;
-			mBuffer = nullptr;
-		}
-	}
-};
-
 struct FResourceLump
 {
 protected:

src/common/filesystem/source/critsec.cpp

@@ -0,0 +1,150 @@
+/*
+**
+**
+**---------------------------------------------------------------------------
+** Copyright 2005-2016 Randy Heit
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+**
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**---------------------------------------------------------------------------
+**
+*/
+
+namespace FileSys {
+
+#ifdef _WIN32
+
+#ifndef _WINNT_
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#endif
+
+class FInternalCriticalSection
+{
+public:
+	void Enter()
+	{
+		AcquireSRWLockExclusive(&CritSec);
+	}
+	void Leave()
+	{
+		ReleaseSRWLockExclusive(&CritSec);
+	}
+private:
+	SRWLOCK CritSec = SRWLOCK_INIT;
+};
+
+
+FInternalCriticalSection *CreateCriticalSection()
+{
+	return new FInternalCriticalSection();
+}
+
+void DeleteCriticalSection(FInternalCriticalSection *c)
+{
+	delete c;
+}
+
+void EnterCriticalSection(FInternalCriticalSection *c)
+{
+	c->Enter();
+}
+
+void LeaveCriticalSection(FInternalCriticalSection *c)
+{
+	c->Leave();
+}
+
+#else
+
+#include "critsec.h"
+
+#include <pthread.h>
+
+class FInternalCriticalSection
+{
+public:
+	FInternalCriticalSection();
+	~FInternalCriticalSection();
+
+	void Enter();
+	void Leave();
+
+private:
+	pthread_mutex_t m_mutex;
+
+};
+
+// TODO: add error handling
+
+FInternalCriticalSection::FInternalCriticalSection()
+{
+	pthread_mutexattr_t attributes;
+	pthread_mutexattr_init(&attributes);
+	pthread_mutexattr_settype(&attributes, PTHREAD_MUTEX_RECURSIVE);
+
+	pthread_mutex_init(&m_mutex, &attributes);
+
+	pthread_mutexattr_destroy(&attributes);
+}
+
+FInternalCriticalSection::~FInternalCriticalSection()
+{
+	pthread_mutex_destroy(&m_mutex);
+}
+
+void FInternalCriticalSection::Enter()
+{
+	pthread_mutex_lock(&m_mutex);
+}
+
+void FInternalCriticalSection::Leave()
+{
+	pthread_mutex_unlock(&m_mutex);
+}
+
+
+FInternalCriticalSection *CreateCriticalSection()
+{
+	return new FInternalCriticalSection();
+}
+
+void DeleteCriticalSection(FInternalCriticalSection *c)
+{
+	delete c;
+}
+
+void EnterCriticalSection(FInternalCriticalSection *c)
+{
+	c->Enter();
+}
+
+void LeaveCriticalSection(FInternalCriticalSection *c)
+{
+	c->Leave();
+} 
+
+#endif
+
+}

src/common/filesystem/source/critsec.h

@@ -0,0 +1,39 @@
+#pragma once
+
+namespace FileSys {
+// System independent critical sections without polluting the namespace with the operating system headers.
+class FInternalCriticalSection;
+FInternalCriticalSection *CreateCriticalSection();
+void DeleteCriticalSection(FInternalCriticalSection *c);
+void EnterCriticalSection(FInternalCriticalSection *c);
+void LeaveCriticalSection(FInternalCriticalSection *c);
+
+// This is just a convenience wrapper around the function interface adjusted to use std::lock_guard
+class FCriticalSection
+{
+public:
+	FCriticalSection()
+	{
+		c = CreateCriticalSection();
+	}
+	
+	~FCriticalSection()
+	{
+		DeleteCriticalSection(c);
+	}
+
+	void lock()
+	{
+		EnterCriticalSection(c);
+	}
+	
+	void unlock()
+	{
+		LeaveCriticalSection(c);
+	}
+
+private:
+	FInternalCriticalSection *c;
+
+};
+}

src/common/filesystem/source/file_7z.cpp

@@ -39,6 +39,8 @@
 #include "resourcefile.h"
 #include "fs_findfile.h"
 #include "unicode.h"
+#include "critsec.h"
+#include <mutex>
 
 
 namespace FileSys {
@@ -171,6 +173,7 @@ class F7ZFile : public FResourceFile
 	friend struct F7ZLump;
 
 	C7zArchive *Archive;
+	FCriticalSection critsec;
 
 public:
 	F7ZFile(const char * filename, FileReader &filer, StringPool* sp);
@@ -318,7 +321,7 @@ FileData F7ZFile::Read(int entry)
 	{
 		auto p = buffer.allocate(Entries[entry].Length);
 		// There is no realistic way to keep multiple references to a 7z file open without massive overhead so to make this thread-safe a mutex is the only option.
-		//std::lock_guard<FCriticalSection> lock(critsec); // activate later
+		std::lock_guard<FCriticalSection> lock(critsec);
 		SRes code = Archive->Extract(Entries[entry].Position, (char*)p);
 		if (code != SZ_OK) buffer.clear();
 	}

src/common/filesystem/source/file_zip.cpp

@@ -67,6 +67,11 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdi
 	case METHOD_BZIP2:
 	case METHOD_LZMA:
 	case METHOD_XZ:
+	case METHOD_IMPLODE_0:
+	case METHOD_IMPLODE_2:
+	case METHOD_IMPLODE_4:
+	case METHOD_IMPLODE_6:
+	case METHOD_SHRINK:
 	{
 		FileReader frz;
 		if (frz.OpenDecompressor(Reader, LumpSize, Method, false, exceptions))
@@ -76,38 +81,6 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdi
 		break;
 	}
 
-	case METHOD_IMPLODE_0:
-	case METHOD_IMPLODE_2:
-	case METHOD_IMPLODE_4:
-	case METHOD_IMPLODE_6:
-	{
-		FZipExploder exploder;
-		if (exploder.Explode((unsigned char*)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize, Method - METHOD_IMPLODE_MIN) == -1)
-		{
-			// decompression failed so zero the cache.
-			memset(Cache, 0, LumpSize);
-		}
-		break;
-	}
-
-	// This can go away once we are done with FResourceLump
-	case METHOD_IMPLODE:
-	{
-		FZipExploder exploder;
-		if (exploder.Explode((unsigned char*)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize, GPFlags) == -1)
-		{
-			// decompression failed so zero the cache.
-			memset(Cache, 0, LumpSize);
-		}
-		break;
-	}
-
-	case METHOD_SHRINK:
-	{
-		ShrinkLoop((unsigned char *)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize);
-		break;
-	}
-
 	default:
 		assert(0);
 		return false;
@@ -115,13 +88,6 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdi
 	return true;
 }
 
-bool FCompressedBuffer::Decompress(char *destbuffer)
-{
-	FileReader mr;
-	mr.OpenMemory(mBuffer, mCompressedSize);
-	return UncompressZipLump(destbuffer, mr, mMethod, mSize, mCompressedSize, 0, false);
-}
-
 //-----------------------------------------------------------------------
 //
 // Finds the central directory end record in the end of the file.

src/common/filesystem/source/files_decompress.cpp

@@ -45,6 +45,8 @@
 #include <stdexcept>
 
 #include "fs_files.h"
+#include "files_internal.h"
+#include "ancientzip.h"
 
 namespace FileSys {
 	using namespace byteswap;
@@ -53,6 +55,7 @@ namespace FileSys {
 class DecompressorBase : public FileReaderInterface
 {
 	bool exceptions = false;
+
 public:
 	// These do not work but need to be defined to satisfy the FileReaderInterface.
 	// They will just error out when called.
@@ -64,6 +67,10 @@ public:
 	void EnableExceptions(bool on) { exceptions = on; }
 
 protected:
+	DecompressorBase()
+	{
+		//seekable = false;
+	}
 	FileReader* File = nullptr;
 	FileReader OwnedFile;
 };
@@ -843,6 +850,7 @@ public:
 
 bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions)
 {
+	FileReaderInterface* fr = nullptr;
 	DecompressorBase* dec = nullptr;
 	try
 	{
@@ -853,7 +861,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 		case METHOD_ZLIB:
 		{
 			auto idec = new DecompressorZ;
-			dec = idec;
+			fr = dec = idec;
 			idec->EnableExceptions(exceptions);
 			if (!idec->Open(p, method == METHOD_DEFLATE))
 			{
@@ -865,7 +873,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 		case METHOD_BZIP2:
 		{
 			auto idec = new DecompressorBZ2;
-			dec = idec;
+			fr = dec = idec;
 			idec->EnableExceptions(exceptions);
 			if (!idec->Open(p))
 			{
@@ -877,7 +885,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 		case METHOD_LZMA:
 		{
 			auto idec = new DecompressorLZMA;
-			dec = idec;
+			fr = dec = idec;
 			idec->EnableExceptions(exceptions);
 			if (!idec->Open(p, length))
 			{
@@ -889,7 +897,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 		case METHOD_XZ:
 		{
 			auto idec = new DecompressorXZ;
-			dec = idec;
+			fr = dec = idec;
 			idec->EnableExceptions(exceptions);
 			if (!idec->Open(p, length))
 			{
@@ -901,7 +909,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 		case METHOD_LZSS:
 		{
 			auto idec = new DecompressorLZSS;
-			dec = idec;
+			fr = dec = idec;
 			idec->EnableExceptions(exceptions);
 			if (!idec->Open(p))
 			{
@@ -911,34 +919,85 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b
 			break;
 		}
 
-		// todo: METHOD_IMPLODE, METHOD_SHRINK
+		// The decoders for these legacy formats can only handle the full data in one go so we have to perform the entire decompression here.
+		case METHOD_IMPLODE:
+		case METHOD_IMPLODE_2:
+		case METHOD_IMPLODE_4:
+		case METHOD_IMPLODE_6:
+		{
+			auto idec = new MemoryArrayReader<FileData>;
+			fr = idec;
+			auto& buffer = idec->GetArray();
+			auto bufr = (uint8_t*)buffer.allocate(length);
+			FZipExploder exploder;
+			if (exploder.Explode(bufr, length, *p, p->GetLength(), method) == -1)
+			{
+				if (exceptions)
+				{
+					throw FileSystemException("DecompressImplode failed");
+				}
+				delete idec;
+				return false;
+			}
+			break;
+		}
+
+		case METHOD_SHRINK:
+		{
+			auto idec = new MemoryArrayReader<FileData>;
+			fr = idec;
+			auto& buffer = idec->GetArray();
+			auto bufr = (uint8_t*)buffer.allocate(length);
+			ShrinkLoop(bufr, length, *p, p->GetLength()); // this never fails.
+			break;
+		}
+
 		default:
 			return false;
 		}
-		if (method & METHOD_TRANSFEROWNER)
+		if (dec)
 		{
-			dec->SetOwnsReader();
+			if (method & METHOD_TRANSFEROWNER)
+			{
+				dec->SetOwnsReader();
+			}
+			dec->Length = length;
 		}
-
-		dec->Length = length;
 		if (!seekable)
 		{
 			Close();
-			mReader = dec;
-			return true;
+			mReader = fr;
 		}
 		else
 		{
-			// todo: create a wrapper. for now this fails
-			delete dec;
-			return false;
+			// create a wrapper that can buffer the content so that seeking is possible
+			mReader = new BufferingReader(fr);
 		}
+		return true;
 	}
 	catch (...)
 	{
-		if (dec) delete dec;
+		if (fr) delete fr;
 		throw;
 	}
 }
 
+
+bool FCompressedBuffer::Decompress(char* destbuffer)
+{
+	FileReader mr;
+	mr.OpenMemory(mBuffer, mCompressedSize);
+	if (mMethod == METHOD_STORED)
+	{
+		return mr.Read(destbuffer, mSize) != mSize;
+	}
+	else
+	{
+		FileReader frz;
+		if (frz.OpenDecompressor(mr, mSize, mMethod, false, false))
+		{
+			return frz.Read(destbuffer, mSize) != mSize;
+		}
+	}
+	return false;
 }

src/common/filesystem/source/files_internal.h

@@ -83,7 +83,7 @@ public:
 		UpdateBuffer();
 	}
 
-	std::vector<uint8_t>& GetArray() { return buf; }
+	T& GetArray() { return buf; }
 
 	void UpdateBuffer()
 	{

src/common/filesystem/source/fs_stringpool.cpp

@@ -108,8 +108,7 @@ void *StringPool::Alloc(size_t size)
 {
 	Block *block;
 
-	size = ((size)+8) & ~7;
-
+	size = (size + 7) & ~7;
 	for (block = TopBlock; block != nullptr; block = block->NextBlock)
 	{
 		void *res = block->Alloc(size);
@@ -124,7 +123,7 @@ void *StringPool::Alloc(size_t size)
 
 const char* StringPool::Strdup(const char* str)
 {
-	char* p = (char*)Alloc(strlen(str));
+	char* p = (char*)Alloc(strlen(str) + 1);
 	strcpy(p, str);
 	return p;
 }