let FileReader::Read return an opaque buffer instead of std::vector.

This can later allow returning a pointer to a static buffer from a cache without creating copies.

src/common/audio/music/i_music.cpp

@@ -337,7 +337,7 @@ static ZMusic_MidiSource GetMIDISource(const char *fn)
 	}
 
 	auto data = wlump.Read();
-	auto source = ZMusic_CreateMIDISource(data.data(), data.size(), type);
+	auto source = ZMusic_CreateMIDISource(data.bytes(), data.size(), type);
 
 	if (source == nullptr)
 	{

src/common/cutscenes/movieplayer.cpp

@@ -153,7 +153,7 @@ class AnmPlayer : public MoviePlayer
 {
 	// This doesn't need its own class type
 	anim_t anim;
-	std::vector<uint8_t> buffer;
+	FileSys::ResourceData buffer;
 	int numframes = 0;
 	int curframe = 1;
 	int frametime = 0;
@@ -173,7 +173,7 @@ public:
 		buffer = fr.ReadPadded(1);
 		fr.Close();
 
-		if (ANIM_LoadAnim(&anim, buffer.data(), buffer.size() - 1) < 0)
+		if (ANIM_LoadAnim(&anim, buffer.bytes(), buffer.size() - 1) < 0)
 		{
 			return;
 		}

src/common/filesystem/include/fs_files.h

@@ -90,6 +90,84 @@ enum
 
 class FileReader;
 
+// an opaque memory buffer to the file's content. Can either own the memory or just point to an external buffer.
+class ResourceData
+{
+	void* memory;
+	size_t length;
+	bool owned;
+
+public:
+	using value_type = uint8_t;
+	ResourceData() { memory = nullptr; length = 0; owned = true; }
+	const void* data() const { return memory; }
+	size_t size() const { return length; }
+	const char* string() const { return (const char*)memory; }
+	const uint8_t* bytes() const { return (const uint8_t*)memory; }
+
+	ResourceData& operator = (const ResourceData& copy)
+	{
+		if (owned && memory) free(memory);
+		length = copy.length;
+		owned = copy.owned;
+		if (owned)
+		{
+			memory = malloc(length);
+			memcpy(memory, copy.memory, length);
+		}
+		else memory = copy.memory;
+		return *this;
+	}
+
+	ResourceData& operator = (ResourceData&& copy) noexcept
+	{
+		if (owned && memory) free(memory);
+		length = copy.length;
+		owned = copy.owned;
+		memory = copy.memory;
+		copy.memory = nullptr;
+		copy.length = 0;
+		copy.owned = true;
+		return *this;
+	}
+
+	ResourceData(const ResourceData& copy)
+	{
+		memory = nullptr;
+		*this = copy;
+	}
+
+	~ResourceData()
+	{
+		if (owned && memory) free(memory);
+	}
+
+	void* allocate(size_t len)
+	{
+		if (!owned) memory = nullptr;
+		length = len;
+		owned = true;
+		memory = realloc(memory, length);
+		return memory;
+	}
+
+	void set(const void* mem, size_t len)
+	{
+		memory = (void*)mem;
+		length = len;
+		owned = false;
+	}
+
+	void clear()
+	{
+		if (owned && memory) free(memory);
+		memory = nullptr;
+		length = 0;
+		owned = true;
+	}
+
+};
+
 class FileReaderInterface
 {
 public:
@@ -171,10 +249,12 @@ public:
 		mReader = nullptr;
 	}
 
-	bool OpenFile(const char *filename, Size start = 0, Size length = -1);
+	bool OpenFile(const char *filename, Size start = 0, Size length = -1, bool buffered = false);
 	bool OpenFilePart(FileReader &parent, Size start, Size length);
 	bool OpenMemory(const void *mem, Size length);	// read directly from the buffer
 	bool OpenMemoryArray(const void *mem, Size length);	// read from a copy of the buffer.
+	bool OpenMemoryArray(std::vector<uint8_t>& data);	// take the given array
+	bool OpenMemoryArray(ResourceData& data);	// take the given array
 	bool OpenMemoryArray(std::function<bool(std::vector<uint8_t>&)> getter);	// read contents to a buffer and return a reader to it
 	bool OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions = false);	// creates a decompressor stream. 'seekable' uses a buffered version so that the Seek and Tell methods can be used.
 
@@ -193,33 +273,34 @@ public:
 		return mReader->Read(buffer, len);
 	}
 
-	std::vector<uint8_t> Read(size_t len)
+	ResourceData Read(size_t len)
 	{
-		std::vector<uint8_t> buffer(len);
+		ResourceData buffer;
 		if (len > 0)
 		{
-			Size length = mReader->Read(&buffer[0], len);
+			Size length = mReader->Read(buffer.allocate(len), len);
-			buffer.resize((size_t)length);
+			if (length < len) buffer.allocate(length);
 		}
 		return buffer;
 	}
 
-	std::vector<uint8_t> Read()
+	ResourceData Read()
 	{
 		return Read(GetLength());
 	}
 
-	std::vector<uint8_t> ReadPadded(size_t padding)
+	ResourceData ReadPadded(size_t padding)
 	{
 		auto len = GetLength();
-		std::vector<uint8_t> buffer(len + padding);
+		ResourceData buffer;
+
 		if (len > 0)
 		{
-			Size length = mReader->Read(&buffer[0], len);
+			auto p = (char*)buffer.allocate(len + padding);
+			Size length = mReader->Read(p, len);
 			if (length < len) buffer.clear();
-			else memset(buffer.data() + len, 0, padding);
+			else memset(p + len, 0, padding);
 		}
-		else buffer[0] = 0;
 		return buffer;
 	}
 
@@ -353,13 +434,13 @@ protected:
 class BufferWriter : public FileWriter
 {
 protected:
-	TArray<unsigned char> mBuffer;
+	std::vector<unsigned char> mBuffer;
 public:
 
 	BufferWriter() {}
 	virtual size_t Write(const void *buffer, size_t len) override;
-	TArray<unsigned char> *GetBuffer() { return &mBuffer; }
+	std::vector<unsigned char> *GetBuffer() { return &mBuffer; }
-	TArray<unsigned char>&& TakeBuffer() { return std::move(mBuffer); }
+	std::vector<unsigned char>&& TakeBuffer() { return std::move(mBuffer); }
 };
 
 }

src/common/filesystem/include/resourcefile.h

@@ -88,8 +88,8 @@ enum ELumpFlags
 // This holds a compresed Zip entry with all needed info to decompress it.
 struct FCompressedBuffer
 {
-	unsigned mSize;
+	size_t mSize;
-	unsigned mCompressedSize;
+	size_t mCompressedSize;
 	int mMethod;
 	int mZipFlags;
 	unsigned mCRC32;

src/common/filesystem/source/files.cpp

@@ -34,6 +34,7 @@
 */
 
 #include <string>
+#include <memory>
 #include "files_internal.h"
 
 namespace FileSys {
@@ -285,7 +286,7 @@ ptrdiff_t MemoryReader::Seek(ptrdiff_t offset, int origin)
 
 ptrdiff_t MemoryReader::Read(void *buffer, ptrdiff_t len)
 {
-	if (len>Length - FilePos) len = Length - FilePos;
+	if (len > Length - FilePos) len = Length - FilePos;
 	if (len<0) len = 0;
 	memcpy(buffer, bufptr + FilePos, len);
 	FilePos += len;
@@ -322,40 +323,37 @@ char *MemoryReader::Gets(char *strbuf, ptrdiff_t len)
 	return strbuf;
 }
 
-//==========================================================================
+int BufferingReader::FillBuffer(size_t newpos)
-//
-// MemoryArrayReader
-//
-// reads data from an array of memory
-//
-//==========================================================================
-
-class MemoryArrayReader : public MemoryReader
 {
-	std::vector<uint8_t> buf;
+	if (newpos > Length) newpos = Length;
-
+	if (newpos < bufferpos) return 0;
-public:
+	auto read = baseReader->Read(&buf[bufferpos], newpos - bufferpos);
-	MemoryArrayReader(const char *buffer, ptrdiff_t length)
+	bufferpos += read;
+	if (bufferpos == Length)
 	{
-		if (length > 0)
+		// we have read the entire file, so delete our data provider.
-		{
+		baseReader.reset();
-			buf.resize(length);
-			memcpy(&buf[0], buffer, length);
-		}
-		UpdateBuffer();
 	}
+	return read == newpos - bufferpos ? 0 : -1;
+}
 
-	std::vector<uint8_t> &GetArray() { return buf; }
+ptrdiff_t BufferingReader::Seek(ptrdiff_t offset, int origin)
-
+{
-	void UpdateBuffer() 
+	if (-1 == MemoryReader::Seek(offset, origin)) return -1;
-	{ 
+	return FillBuffer(FilePos);
-		bufptr = (const char*)buf.data();
+}
-		FilePos = 0;
-		Length = buf.size();
-	}
-};
 
+ptrdiff_t BufferingReader::Read(void* buffer, ptrdiff_t len)
+{
+	if (FillBuffer(FilePos + len) < 0) return 0;
+	return MemoryReader::Read(buffer, len);
+}
 
+char* BufferingReader::Gets(char* strbuf, ptrdiff_t len)
+{
+	if (FillBuffer(FilePos + len) < 0) return nullptr;
+	return MemoryReader::Gets(strbuf, len);
+}
 
 //==========================================================================
 //
@@ -365,7 +363,7 @@ public:
 //
 //==========================================================================
 
-bool FileReader::OpenFile(const char *filename, FileReader::Size start, FileReader::Size length)
+bool FileReader::OpenFile(const char *filename, FileReader::Size start, FileReader::Size length, bool buffered)
 {
 	auto reader = new StdFileReader;
 	if (!reader->Open(filename, start, length))
@@ -374,7 +372,8 @@ bool FileReader::OpenFile(const char *filename, FileReader::Size start, FileRead
 		return false;
 	}
 	Close();
-	mReader = reader;
+	if (buffered) mReader = new BufferingReader(reader);
+	else mReader = reader;
 	return true;
 }
 
@@ -396,13 +395,27 @@ bool FileReader::OpenMemory(const void *mem, FileReader::Size length)
 bool FileReader::OpenMemoryArray(const void *mem, FileReader::Size length)
 {
 	Close();
-	mReader = new MemoryArrayReader((const char *)mem, length);
+	mReader = new MemoryArrayReader<std::vector<uint8_t>>((const char *)mem, length);
+	return true;
+}
+
+bool FileReader::OpenMemoryArray(std::vector<uint8_t>& data)
+{
+	Close();
+	if (data.size() > 0) mReader = new MemoryArrayReader<std::vector<uint8_t>>(data);
+	return true;
+}
+
+bool FileReader::OpenMemoryArray(ResourceData& data)
+{
+	Close();
+	if (data.size() > 0) mReader = new MemoryArrayReader<ResourceData>(data);
 	return true;
 }
 
 bool FileReader::OpenMemoryArray(std::function<bool(std::vector<uint8_t>&)> getter)
 {
-	auto reader = new MemoryArrayReader(nullptr, 0);
+	auto reader = new MemoryArrayReader<std::vector<uint8_t>>(nullptr, 0);
 	if (getter(reader->GetArray()))
 	{
 		Close();
@@ -494,7 +507,8 @@ size_t FileWriter::Printf(const char *fmt, ...)
 
 size_t BufferWriter::Write(const void *buffer, size_t len)
 {
-	unsigned int ofs = mBuffer.Reserve((unsigned)len);
+	size_t ofs = mBuffer.size();
+	mBuffer.resize(ofs + len);
 	memcpy(&mBuffer[ofs], buffer, len);
 	return len;
 }

src/common/filesystem/source/files_internal.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <memory>
 #include "fs_files.h"
 
 namespace FileSys {
@@ -28,4 +29,71 @@ public:
 	virtual const char *GetBuffer() const override { return bufptr; }
 };
 
+class BufferingReader : public MemoryReader
+{
+	std::vector<uint8_t> buf;
+	std::unique_ptr<FileReaderInterface> baseReader;
+	ptrdiff_t bufferpos = 0;
+
+	int FillBuffer(size_t newpos);
+public:
+	BufferingReader(FileReaderInterface* base)
+		: baseReader(base)
+	{
+	}
+
+	ptrdiff_t Seek(ptrdiff_t offset, int origin) override;
+	ptrdiff_t Read(void* buffer, ptrdiff_t len) override;
+	char* Gets(char* strbuf, ptrdiff_t len) override;
+};
+
+//==========================================================================
+//
+// MemoryArrayReader
+//
+// reads data from an array of memory
+//
+//==========================================================================
+
+template<class T>
+class MemoryArrayReader : public MemoryReader
+{
+	T buf;
+
+public:
+	MemoryArrayReader()
+	{
+		FilePos = 0;
+		Length = 0;
+	}
+
+	MemoryArrayReader(const char* buffer, ptrdiff_t length)
+	{
+		if (length > 0)
+		{
+			buf.resize(length);
+			memcpy(&buf[0], buffer, length);
+		}
+		UpdateBuffer();
+	}
+
+	MemoryArrayReader(T& buffer)
+	{
+		buf = std::move(buffer);
+		UpdateBuffer();
+	}
+
+	std::vector<uint8_t>& GetArray() { return buf; }
+
+	void UpdateBuffer()
+	{
+		bufptr = (const char*)buf.data();
+		FilePos = 0;
+		Length = buf.size();
+	}
+};
+
+bool OpenMemoryArray(std::vector<uint8_t>& data);	// take the given array
+
+
 }

src/common/textures/formats/anmtexture.cpp

@@ -72,9 +72,10 @@ FImageSource *AnmImage_TryCreate(FileReader & file, int lumpnum)
 	if (memcmp(check, "LPF ", 4)) return nullptr;
 	file.Seek(0, FileReader::SeekSet);
 	auto buffer = file.ReadPadded(1);
+	if (buffer.size() < 4) return nullptr;
 
 	std::unique_ptr<anim_t> anim = std::make_unique<anim_t>(); // note that this struct is very large and should not go onto the stack!
-	if (ANIM_LoadAnim(anim.get(), buffer.data(), buffer.size() - 1) < 0)
+	if (ANIM_LoadAnim(anim.get(), buffer.bytes(), buffer.size() - 1) < 0)
 	{
 		return nullptr;
 	}

src/common/textures/formats/pcxtexture.cpp

@@ -161,7 +161,7 @@ void FPCXTexture::ReadPCX1bit (uint8_t *dst, FileReader & lump, PCXHeader *hdr)
 	uint8_t rle_value = 0;
 
 	auto srcp = lump.Read(lump.GetLength() - sizeof(PCXHeader));
-	uint8_t * src = srcp.data();
+	const uint8_t * src = srcp.bytes();
 
 	for (y = 0; y < Height; ++y)
 	{
@@ -211,7 +211,7 @@ void FPCXTexture::ReadPCX4bits (uint8_t *dst, FileReader & lump, PCXHeader *hdr)
 	TArray<uint8_t> colorIndex(Width, true);
 
 	auto srcp = lump.Read(lump.GetLength() - sizeof(PCXHeader));
-	uint8_t * src = srcp.data();
+	const uint8_t * src = srcp.bytes();
 
 	for (y = 0; y < Height; ++y)
 	{
@@ -265,7 +265,7 @@ void FPCXTexture::ReadPCX8bits (uint8_t *dst, FileReader & lump, PCXHeader *hdr)
 	int y, bytes;
 
 	auto srcp = lump.Read(lump.GetLength() - sizeof(PCXHeader));
-	uint8_t * src = srcp.data();
+	const uint8_t * src = srcp.bytes();
 
 	for (y = 0; y < Height; ++y)
 	{
@@ -306,7 +306,7 @@ void FPCXTexture::ReadPCX24bits (uint8_t *dst, FileReader & lump, PCXHeader *hdr
 	int bytes;
 
 	auto srcp = lump.Read(lump.GetLength() - sizeof(PCXHeader));
-	uint8_t * src = srcp.data();
+	const uint8_t * src = srcp.bytes();
 
 	for (y = 0; y < Height; ++y)
 	{

src/common/textures/formats/webptexture.cpp

@@ -65,9 +65,9 @@ FImageSource *WebPImage_TryCreate(FileReader &file, int lumpnum)
 	file.Seek(0, FileReader::SeekSet);
 	auto bytes = file.Read();
 
-	if (WebPGetInfo(bytes.data(), bytes.size(), &width, &height))
+	if (WebPGetInfo(bytes.bytes(), bytes.size(), &width, &height))
 	{
-		WebPData data{ bytes.data(), bytes.size() };
+		WebPData data{ bytes.bytes(), bytes.size() };
 		WebPData chunk_data;
 		auto mux = WebPMuxCreate(&data, 0);
 		if (mux)

src/g_game.cpp

@@ -2427,7 +2427,7 @@ void G_DoSaveGame (bool okForQuicksave, bool forceQuicksave, FString filename, c
 	}
 
 	auto picdata = savepic.GetBuffer();
-	FCompressedBuffer bufpng = { picdata->Size(), picdata->Size(), FileSys::METHOD_STORED, 0, static_cast<unsigned int>(crc32(0, &(*picdata)[0], picdata->Size())), (char*)&(*picdata)[0] };
+	FCompressedBuffer bufpng = { picdata->size(), picdata->size(), FileSys::METHOD_STORED, 0, static_cast<unsigned int>(crc32(0, &(*picdata)[0], picdata->size())), (char*)&(*picdata)[0] };
 
 	savegame_content.Push(bufpng);
 	savegame_filenames.Push("savepic.png");

src/m_misc.cpp

@@ -98,7 +98,7 @@ void M_FindResponseFile (void)
 		else
 		{
 			char	**argv;
-			std::vector<uint8_t> file;
+			FileSys::ResourceData file;
 			int		argc = 0;
 			int 	size;
 			size_t	argsize = 0;
@@ -118,9 +118,8 @@ void M_FindResponseFile (void)
 				{
 					Printf ("Found response file %s!\n", Args->GetArg(i) + 1);
 					size = (int)fr.GetLength();
-					file = fr.Read (size);
+					file = fr.ReadPadded(1);
-					file.push_back(0);
+					argsize = ParseCommandLine (file.string(), &argc, nullptr);
-					argsize = ParseCommandLine ((char*)file.data(), &argc, NULL);
 				}
 			}
 			else
@@ -132,7 +131,7 @@ void M_FindResponseFile (void)
 			{
 				argv = (char **)M_Malloc (argc*sizeof(char *) + argsize);
 				argv[0] = (char *)argv + argc*sizeof(char *);
-				ParseCommandLine ((char*)file.data(), NULL, argv);
+				ParseCommandLine (file.string(), nullptr, argv);
 
 				// Create a new argument vector
 				FArgs *newargs = new FArgs;

src/maploader/glnodes.cpp

@@ -209,7 +209,7 @@ bool MapLoader::LoadGLVertexes(FileReader &lump)
 	auto gllen=lump.GetLength();
 	if (gllen < 4)
 		return false;
-	auto gldata = glbuf.data();
+	auto gldata = glbuf.bytes();
 
 	if (*(int *)gldata == gNd5) 
 	{
@@ -294,7 +294,7 @@ bool MapLoader::LoadGLSegs(FileReader &lump)
 		segs.Alloc(numsegs);
 		memset(&segs[0],0,sizeof(seg_t)*numsegs);
 			
-		glseg_t * ml = (glseg_t*)data.data();
+		auto ml = (const glseg_t*)data.data();
 		for(i = 0; i < numsegs; i++)
 		{		
 			// check for gl-vertices
@@ -324,21 +324,21 @@ bool MapLoader::LoadGLSegs(FileReader &lump)
 				ldef = &Level->lines[lineidx];
 				segs[i].linedef = ldef;	
 					
-				ml->side=LittleShort(ml->side);
+				auto side=LittleShort(ml->side);
-				if (ml->side > 1) 
+				if (side > 1) 
 					return false;
-				segs[i].sidedef = ldef->sidedef[ml->side];
+				segs[i].sidedef = ldef->sidedef[side];
-				if (ldef->sidedef[ml->side] != nullptr)
+				if (ldef->sidedef[side] != nullptr)
 				{
-					segs[i].frontsector = ldef->sidedef[ml->side]->sector;
+					segs[i].frontsector = ldef->sidedef[side]->sector;
 				}
 				else
 				{
 					segs[i].frontsector = nullptr;
 				}
-				if (ldef->flags & ML_TWOSIDED && ldef->sidedef[ml->side^1] != nullptr)
+				if (ldef->flags & ML_TWOSIDED && ldef->sidedef[side^1] != nullptr)
 				{
-					segs[i].backsector = ldef->sidedef[ml->side^1]->sector;
+					segs[i].backsector = ldef->sidedef[side^1]->sector;
 				}
 				else
 				{
@@ -365,7 +365,7 @@ bool MapLoader::LoadGLSegs(FileReader &lump)
 		segs.Alloc(numsegs);
 		memset(&segs[0],0,sizeof(seg_t)*numsegs);
 			
-		glseg3_t * ml = (glseg3_t*)(data.data() + (format5? 0:4));
+		const glseg3_t * ml = (const glseg3_t*)(data.bytes() + (format5? 0:4));
 		for(i = 0; i < numsegs; i++)
 		{							// check for gl-vertices
 			const unsigned v1idx = checkGLVertex3(LittleLong(ml->v1));
@@ -395,21 +395,21 @@ bool MapLoader::LoadGLSegs(FileReader &lump)
 				segs[i].linedef = ldef;
 	
 					
-				ml->side=LittleShort(ml->side);
+				auto side=LittleShort(ml->side);
-				if (ml->side > 1)
+				if (side > 1)
 					return false;
-				segs[i].sidedef = ldef->sidedef[ml->side];
+				segs[i].sidedef = ldef->sidedef[side];
-				if (ldef->sidedef[ml->side] != nullptr)
+				if (ldef->sidedef[side] != nullptr)
 				{
-					segs[i].frontsector = ldef->sidedef[ml->side]->sector;
+					segs[i].frontsector = ldef->sidedef[side]->sector;
 				}
 				else
 				{
 					segs[i].frontsector = nullptr;
 				}
-				if (ldef->flags & ML_TWOSIDED && ldef->sidedef[ml->side^1] != nullptr)
+				if (ldef->flags & ML_TWOSIDED && ldef->sidedef[side^1] != nullptr)
 				{
-					segs[i].backsector = ldef->sidedef[ml->side^1]->sector;
+					segs[i].backsector = ldef->sidedef[side^1]->sector;
 				}
 				else
 				{
@@ -456,7 +456,7 @@ bool MapLoader::LoadGLSubsectors(FileReader &lump)
 	
 	if (!format5 && memcmp(datab.data(), "gNd3", 4))
 	{
-		mapsubsector_t * data = (mapsubsector_t*) datab.data();
+		auto data = (const mapsubsector_t*) datab.data();
 		numsubsectors /= sizeof(mapsubsector_t);
 		Level->subsectors.Alloc(numsubsectors);
 		auto &subsectors = Level->subsectors;
@@ -478,7 +478,7 @@ bool MapLoader::LoadGLSubsectors(FileReader &lump)
 	}
 	else
 	{
-		gl3_mapsubsector_t * data = (gl3_mapsubsector_t*) (datab.data()+(format5? 0:4));
+		auto data = (const gl3_mapsubsector_t*) (datab.bytes()+(format5? 0:4));
 		numsubsectors /= sizeof(gl3_mapsubsector_t);
 		Level->subsectors.Alloc(numsubsectors);
 		auto &subsectors = Level->subsectors;