From 796890bb985212e211c778012c073e98d1e2de54 Mon Sep 17 00:00:00 2001
From: Magnus Norddahl <dpjudas@users.noreply.github.com>
Date: Mon, 13 Sep 2021 21:58:44 +0200
Subject: [PATCH] Limit texture upload transfer buffers to roughly 64 MB

---
 .../vulkan/system/vk_framebuffer.cpp          | 37 +++++++++++--------
 .../rendering/vulkan/system/vk_framebuffer.h  | 13 +++++--
 .../vulkan/textures/vk_hwtexture.cpp          |  7 +++-
 3 files changed, 38 insertions(+), 19 deletions(-)

diff --git a/src/common/rendering/vulkan/system/vk_framebuffer.cpp b/src/common/rendering/vulkan/system/vk_framebuffer.cpp
index 5bce012243..4d08212ecd 100644
--- a/src/common/rendering/vulkan/system/vk_framebuffer.cpp
+++ b/src/common/rendering/vulkan/system/vk_framebuffer.cpp
@@ -209,15 +209,21 @@ void VulkanFrameBuffer::Update()
 	Super::Update();
 }
 
-void VulkanFrameBuffer::DeleteFrameObjects()
+void VulkanFrameBuffer::DeleteFrameObjects(bool uploadOnly)
 {
-	FrameDeleteList.Images.clear();
-	FrameDeleteList.ImageViews.clear();
-	FrameDeleteList.Framebuffers.clear();
-	FrameDeleteList.Buffers.clear();
-	FrameDeleteList.Descriptors.clear();
-	FrameDeleteList.DescriptorPools.clear();
-	FrameDeleteList.CommandBuffers.clear();
+	FrameTextureUpload.Buffers.clear();
+	FrameTextureUpload.TotalSize = 0;
+
+	if (!uploadOnly)
+	{
+		FrameDeleteList.Images.clear();
+		FrameDeleteList.ImageViews.clear();
+		FrameDeleteList.Framebuffers.clear();
+		FrameDeleteList.Buffers.clear();
+		FrameDeleteList.Descriptors.clear();
+		FrameDeleteList.DescriptorPools.clear();
+		FrameDeleteList.CommandBuffers.clear();
+	}
 }
 
 void VulkanFrameBuffer::FlushCommands(VulkanCommandBuffer **commands, size_t count, bool finish, bool lastsubmit)
@@ -251,11 +257,12 @@ void VulkanFrameBuffer::FlushCommands(VulkanCommandBuffer **commands, size_t cou
 	mNextSubmit++;
 }
 
-void VulkanFrameBuffer::FlushCommands(bool finish, bool lastsubmit)
+void VulkanFrameBuffer::FlushCommands(bool finish, bool lastsubmit, bool uploadOnly)
 {
-	mRenderState->EndRenderPass();
+	if (!uploadOnly)
+		mRenderState->EndRenderPass();
 
-	if (mDrawCommands || mTransferCommands)
+	if ((!uploadOnly && mDrawCommands) || mTransferCommands)
 	{
 		VulkanCommandBuffer *commands[2];
 		size_t count = 0;
@@ -267,7 +274,7 @@ void VulkanFrameBuffer::FlushCommands(bool finish, bool lastsubmit)
 			FrameDeleteList.CommandBuffers.push_back(std::move(mTransferCommands));
 		}
 
-		if (mDrawCommands)
+		if (!uploadOnly && mDrawCommands)
 		{
 			mDrawCommands->end();
 			commands[count++] = mDrawCommands.get();
@@ -280,7 +287,7 @@ void VulkanFrameBuffer::FlushCommands(bool finish, bool lastsubmit)
 	}
 }
 
-void VulkanFrameBuffer::WaitForCommands(bool finish)
+void VulkanFrameBuffer::WaitForCommands(bool finish, bool uploadOnly)
 {
 	if (finish)
 	{
@@ -292,7 +299,7 @@ void VulkanFrameBuffer::WaitForCommands(bool finish)
 			mPostprocess->DrawPresentTexture(mOutputLetterbox, true, false);
 	}
 
-	FlushCommands(finish, true);
+	FlushCommands(finish, true, uploadOnly);
 
 	if (finish)
 	{
@@ -310,7 +317,7 @@ void VulkanFrameBuffer::WaitForCommands(bool finish)
 		vkResetFences(device->device, numWaitFences, mSubmitWaitFences);
 	}
 
-	DeleteFrameObjects();
+	DeleteFrameObjects(uploadOnly);
 	mNextSubmit = 0;
 
 	if (finish)
diff --git a/src/common/rendering/vulkan/system/vk_framebuffer.h b/src/common/rendering/vulkan/system/vk_framebuffer.h
index 9c5e5bf2d3..365278acb8 100644
--- a/src/common/rendering/vulkan/system/vk_framebuffer.h
+++ b/src/common/rendering/vulkan/system/vk_framebuffer.h
@@ -37,7 +37,7 @@ public:
 	VkRenderBuffers *GetBuffers() { return mActiveRenderBuffers; }
 	FRenderState* RenderState() override;
 
-	void FlushCommands(bool finish, bool lastsubmit = false);
+	void FlushCommands(bool finish, bool lastsubmit = false, bool uploadOnly = false);
 
 	unsigned int GetLightBufferBlockSize() const;
 
@@ -64,6 +64,12 @@ public:
 		std::vector<std::unique_ptr<VulkanCommandBuffer>> CommandBuffers;
 	} FrameDeleteList;
 
+	struct
+	{
+		std::vector<std::unique_ptr<VulkanBuffer>> Buffers;
+		size_t TotalSize = 0;
+	} FrameTextureUpload;
+
 	VulkanFrameBuffer(void *hMonitor, bool fullscreen, VulkanDevice *dev);
 	~VulkanFrameBuffer();
 	bool IsVulkan() override { return true; }
@@ -103,7 +109,8 @@ public:
 
 	void Draw2D() override;
 
-	void WaitForCommands(bool finish) override;
+	void WaitForCommands(bool finish) override { WaitForCommands(finish, false); }
+	void WaitForCommands(bool finish, bool uploadOnly);
 
 	void PushGroup(const FString &name);
 	void PopGroup();
@@ -116,7 +123,7 @@ private:
 	void PrintStartupLog();
 	void CreateFanToTrisIndexBuffer();
 	void CopyScreenToBuffer(int w, int h, uint8_t *data) override;
-	void DeleteFrameObjects();
+	void DeleteFrameObjects(bool uploadOnly = false);
 	void FlushCommands(VulkanCommandBuffer **commands, size_t count, bool finish, bool lastsubmit);
 
 	std::unique_ptr<VkShaderManager> mShaderManager;
diff --git a/src/common/rendering/vulkan/textures/vk_hwtexture.cpp b/src/common/rendering/vulkan/textures/vk_hwtexture.cpp
index 4360fc06ee..276f9fd091 100644
--- a/src/common/rendering/vulkan/textures/vk_hwtexture.cpp
+++ b/src/common/rendering/vulkan/textures/vk_hwtexture.cpp
@@ -203,9 +203,14 @@ void VkHardwareTexture::CreateTexture(int w, int h, int pixelsize, VkFormat form
 	region.imageExtent.height = h;
 	cmdbuffer->copyBufferToImage(stagingBuffer->buffer, mImage.Image->image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &region);
 
-	fb->FrameDeleteList.Buffers.push_back(std::move(stagingBuffer));
+	fb->FrameTextureUpload.Buffers.push_back(std::move(stagingBuffer));
 
 	if (mipmap) mImage.GenerateMipmaps(cmdbuffer);
+
+	// If we queued more than 64 MB of data already: wait until the uploads finish before continuing
+	fb->FrameTextureUpload.TotalSize += totalSize;
+	if (fb->FrameTextureUpload.TotalSize > 64 * 1024 * 1024)
+		fb->WaitForCommands(false, true);
 }
 
 int VkHardwareTexture::GetMipLevels(int w, int h)