- add support for flushing commands during drawing

src/rendering/vulkan/system/vk_buffers.cpp

@@ -128,7 +128,7 @@ void VKBuffer::Resize(size_t newsize)
 
 	// Transfer data from old to new
 	fb->GetTransferCommands()->copyBuffer(oldBuffer.get(), mBuffer.get(), 0, 0, oldsize);
-	fb->SubmitCommands(false);
+	fb->WaitForCommands(false);
 
 	// Fetch pointer to new buffer
 	map = mBuffer->Map(0, newsize);

src/rendering/vulkan/system/vk_framebuffer.cpp

@@ -80,6 +80,12 @@ VulkanFrameBuffer::VulkanFrameBuffer(void *hMonitor, bool fullscreen, VulkanDevi
 	mRenderFinishedSemaphore.reset(new VulkanSemaphore(device));
 	mRenderFinishedFence.reset(new VulkanFence(device));
 
+	for (auto &semaphore : mSubmitSemaphore)
+		semaphore.reset(new VulkanSemaphore(device));
+
+	for (auto &fence : mSubmitFence)
+		fence.reset(new VulkanFence(device));
+
 	InitPalette();
 }
 
@@ -120,7 +126,6 @@ void VulkanFrameBuffer::InitializeState()
 	uniformblockalignment = (unsigned int)device->PhysicalDevice.Properties.limits.minUniformBufferOffsetAlignment;
 	maxuniformblock = device->PhysicalDevice.Properties.limits.maxUniformBufferRange;
 
-	mTransferSemaphore.reset(new VulkanSemaphore(device));
 	mCommandPool.reset(new VulkanCommandPool(device, device->graphicsFamily));
 
 	mScreenBuffers.reset(new VkRenderBuffers());
@@ -170,7 +175,7 @@ void VulkanFrameBuffer::Update()
 
 	Flush3D.Unclock();
 
-	SubmitCommands(true);
+	WaitForCommands(true);
 
 	Super::Update();
 }
@@ -181,9 +186,49 @@ void VulkanFrameBuffer::DeleteFrameObjects()
 	FrameDeleteList.ImageViews.clear();
 	FrameDeleteList.Buffers.clear();
 	FrameDeleteList.Descriptors.clear();
+	FrameDeleteList.CommandBuffers.clear();
 }
 
-void VulkanFrameBuffer::SubmitCommands(bool finish)
+void VulkanFrameBuffer::FlushCommands()
+{
+	if (mDrawCommands || mTransferCommands)
+	{
+		int currentIndex = nextSubmitQueue % submitQueueSize;
+
+		if (nextSubmitQueue >= submitQueueSize)
+		{
+			vkWaitForFences(device->device, 1, &mSubmitFence[currentIndex]->fence, VK_TRUE, std::numeric_limits<uint64_t>::max());
+			vkResetFences(device->device, 1, &mSubmitFence[currentIndex]->fence);
+		}
+
+		QueueSubmit submit;
+
+		if (mTransferCommands)
+		{
+			mTransferCommands->end();
+			submit.addCommandBuffer(mTransferCommands.get());
+
+			FrameDeleteList.CommandBuffers.push_back(std::move(mTransferCommands));
+		}
+
+		if (mDrawCommands)
+		{
+			mDrawCommands->end();
+			submit.addCommandBuffer(mDrawCommands.get());
+
+			FrameDeleteList.CommandBuffers.push_back(std::move(mDrawCommands));
+		}
+
+		if (nextSubmitQueue > 0)
+			submit.addWait(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, mSubmitSemaphore[(nextSubmitQueue - 1) % submitQueueSize].get());
+
+		submit.addSignal(mSubmitSemaphore[currentIndex].get());
+		submit.execute(device, device->graphicsQueue, mSubmitFence[currentIndex].get());
+		nextSubmitQueue++;
+	}
+}
+
+void VulkanFrameBuffer::WaitForCommands(bool finish)
 {
 	if (finish)
 	{
@@ -195,31 +240,21 @@ void VulkanFrameBuffer::SubmitCommands(bool finish)
 			mPostprocess->DrawPresentTexture(mOutputLetterbox, true, true);
 	}
 
-	if (mTransferCommands)
+	FlushCommands();
-	{
-		mTransferCommands->end();
-
-		QueueSubmit submit;
-		submit.addCommandBuffer(mTransferCommands.get());
-		submit.addSignal(mTransferSemaphore.get());
-		submit.execute(device, device->graphicsQueue);
-	}
 
 	QueueSubmit submit;
-	if (mDrawCommands)
+
+	if (nextSubmitQueue > 0)
 	{
-		mDrawCommands->end();
+		submit.addWait(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, mSubmitSemaphore[(nextSubmitQueue - 1) % submitQueueSize].get());
-		submit.addCommandBuffer(mDrawCommands.get());
-	}
-	if (mTransferCommands)
-	{
-		submit.addWait(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, mTransferSemaphore.get());
 	}
+
 	if (finish && presentImageIndex != 0xffffffff)
 	{
 		submit.addWait(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, mSwapChainImageAvailableSemaphore.get());
 		submit.addSignal(mRenderFinishedSemaphore.get());
 	}
+
 	submit.execute(device, device->graphicsQueue, mRenderFinishedFence.get());
 
 	if (finish)
@@ -230,11 +265,17 @@ void VulkanFrameBuffer::SubmitCommands(bool finish)
 			swapChain->QueuePresent(presentImageIndex, mRenderFinishedSemaphore.get());
 	}
 
-	vkWaitForFences(device->device, 1, &mRenderFinishedFence->fence, VK_TRUE, std::numeric_limits<uint64_t>::max());
+	VkFence waitFences[submitQueueSize + 1];
-	vkResetFences(device->device, 1, &mRenderFinishedFence->fence);
+	waitFences[0] = mRenderFinishedFence->fence;
-	mDrawCommands.reset();
+	for (int i = 0; i < submitQueueSize; i++)
-	mTransferCommands.reset();
+		waitFences[i + 1] = mSubmitFence[i]->fence;
+
+	int numWaitFences = 1 + MIN(nextSubmitQueue, (int)submitQueueSize);
+	vkWaitForFences(device->device, numWaitFences, waitFences, VK_TRUE, std::numeric_limits<uint64_t>::max());
+	vkResetFences(device->device, numWaitFences, waitFences);
+
 	DeleteFrameObjects();
+	nextSubmitQueue = 0;
 
 	if (finish)
 	{
@@ -257,8 +298,7 @@ void VulkanFrameBuffer::WriteSavePic(player_t *player, FileWriter *file, int wid
 		bounds.height = height;
 
 		// we must be sure the GPU finished reading from the buffer before we fill it with new data.
-		if (mDrawCommands)
+		WaitForCommands(false);
-			SubmitCommands(false);
 
 		// Switch to render buffers dimensioned for the savepic
 		mActiveRenderBuffers = mSaveBuffers.get();
@@ -696,7 +736,7 @@ void VulkanFrameBuffer::CopyScreenToBuffer(int w, int h, void *data)
 	GetDrawCommands()->copyImageToBuffer(image->image, layout, staging->buffer, 1, &region);
 
 	// Submit command buffers and wait for device to finish the work
-	SubmitCommands(false);
+	WaitForCommands(false);
 
 	// Map and convert from rgba8 to rgb8
 	uint8_t *dest = (uint8_t*)data;

src/rendering/vulkan/system/vk_framebuffer.h

@@ -34,6 +34,8 @@ public:
 	VkPostprocess *GetPostprocess() { return mPostprocess.get(); }
 	VkRenderBuffers *GetBuffers() { return mActiveRenderBuffers; }
 
+	void FlushCommands();
+
 	unsigned int GetLightBufferBlockSize() const;
 
 	template<typename T>
@@ -57,6 +59,7 @@ public:
 		std::vector<std::unique_ptr<VulkanImageView>> ImageViews;
 		std::vector<std::unique_ptr<VulkanBuffer>> Buffers;
 		std::vector<std::unique_ptr<VulkanDescriptorSet>> Descriptors;
+		std::vector<std::unique_ptr<VulkanCommandBuffer>> CommandBuffers;
 	} FrameDeleteList;
 
 	std::unique_ptr<SWSceneDrawer> swdrawer;
@@ -97,7 +100,7 @@ public:
 
 	void Draw2D() override;
 
-	void SubmitCommands(bool finish);
+	void WaitForCommands(bool finish);
 
 private:
 	sector_t *RenderViewpoint(FRenderViewpoint &mainvp, AActor * camera, IntRect * bounds, float fov, float ratio, float fovratio, bool mainview, bool toscreen);
@@ -117,10 +120,15 @@ private:
 	std::unique_ptr<VkRenderPassManager> mRenderPassManager;
 	std::unique_ptr<VulkanCommandPool> mCommandPool;
 	std::unique_ptr<VulkanCommandBuffer> mTransferCommands;
-	std::unique_ptr<VulkanCommandBuffer> mDrawCommands;
-	std::unique_ptr<VulkanSemaphore> mTransferSemaphore;
 	std::unique_ptr<VkRenderState> mRenderState;
 
+	std::unique_ptr<VulkanCommandBuffer> mDrawCommands;
+
+	enum { submitQueueSize = 8};
+	std::unique_ptr<VulkanSemaphore> mSubmitSemaphore[submitQueueSize];
+	std::unique_ptr<VulkanFence> mSubmitFence[submitQueueSize];
+	int nextSubmitQueue = 0;
+
 	std::unique_ptr<VulkanSemaphore> mSwapChainImageAvailableSemaphore;
 	std::unique_ptr<VulkanSemaphore> mRenderFinishedSemaphore;
 	std::unique_ptr<VulkanFence> mRenderFinishedFence;