- add vk_submit_multithread for doing command buffer submit calls on a worker thread

- add vk_submit_size for testing various command buffer sizes before flushing them
- add submitted command buffer count to renderstats

src/rendering/hwrenderer/utility/hw_clock.cpp

@@ -54,7 +54,7 @@ glcycle_t twoD, Flush3D;
 glcycle_t MTWait, WTTotal;
 int vertexcount, flatvertices, flatprimitives;
 
-int rendered_lines,rendered_flats,rendered_sprites,render_vertexsplit,render_texsplit,rendered_decals, rendered_portals;
+int rendered_lines,rendered_flats,rendered_sprites,render_vertexsplit,render_texsplit,rendered_decals, rendered_portals, rendered_commandbuffers;
 int iter_dlightf, iter_dlight, draw_dlight, draw_dlightf;
 
 void ResetProfilingData()
@@ -113,8 +113,8 @@ static void AppendRenderStats(FString &out)
 {
 	out.AppendFormat("Walls: %d (%d splits, %d t-splits, %d vertices)\n"
 		"Flats: %d (%d primitives, %d vertices)\n"
-		"Sprites: %d, Decals=%d, Portals: %d\n",
+		"Sprites: %d, Decals=%d, Portals: %d, Command buffers: %d\n",
-		rendered_lines, render_vertexsplit, render_texsplit, vertexcount, rendered_flats, flatprimitives, flatvertices, rendered_sprites,rendered_decals, rendered_portals );
+		rendered_lines, render_vertexsplit, render_texsplit, vertexcount, rendered_flats, flatprimitives, flatvertices, rendered_sprites,rendered_decals, rendered_portals, rendered_commandbuffers );
 }
 
 static void AppendLightStats(FString &out)

src/rendering/vulkan/renderer/vk_renderstate.cpp

@@ -17,6 +17,8 @@
 #include "hwrenderer/data/hw_viewpointbuffer.h"
 #include "hwrenderer/data/shaderuniforms.h"
 
+CVAR(Int, vk_submit_size, 1000, 0);
+
 VkRenderState::VkRenderState()
 {
 	mIdentityMatrix.loadIdentity();
@@ -161,7 +163,7 @@ void VkRenderState::EnableLineSmooth(bool on)
 void VkRenderState::Apply(int dt)
 {
 	mApplyCount++;
-	if (mApplyCount == 1000)
+	if (mApplyCount >= vk_submit_size)
 	{
 		EndRenderPass();
 		GetVulkanFrameBuffer()->FlushCommands();

src/rendering/vulkan/system/vk_framebuffer.cpp

@@ -68,7 +68,11 @@ EXTERN_CVAR(Int, screenblocks)
 EXTERN_CVAR(Bool, cl_capfps)
 EXTERN_CVAR(Bool, gl_no_skyclear)
 
+CVAR(Bool, vk_submit_multithread, false, 0)
+
 extern bool NoInterpolateView;
+extern int rendered_commandbuffers;
+int current_rendered_commandbuffers;
 
 VulkanFrameBuffer::VulkanFrameBuffer(void *hMonitor, bool fullscreen, VulkanDevice *dev) : 
 	Super(hMonitor, fullscreen) 
@@ -91,6 +95,8 @@ VulkanFrameBuffer::VulkanFrameBuffer(void *hMonitor, bool fullscreen, VulkanDevi
 
 VulkanFrameBuffer::~VulkanFrameBuffer()
 {
+	StopSubmitThread();
+
 	// All descriptors must be destroyed before the descriptor pool in renderpass manager is destroyed
 	for (VkHardwareTexture *cur = VkHardwareTexture::First; cur; cur = cur->Next)
 		cur->Reset();
@@ -189,42 +195,112 @@ void VulkanFrameBuffer::DeleteFrameObjects()
 	FrameDeleteList.CommandBuffers.clear();
 }
 
+void VulkanFrameBuffer::StartSubmitThread()
+{
+	if (!mSubmitThread.joinable())
+	{
+		mSubmitThread = std::thread([this] { SubmitThreadMain(); });
+	}
+}
+
+void VulkanFrameBuffer::StopSubmitThread()
+{
+	if (mSubmitThread.joinable())
+	{
+		std::unique_lock<std::mutex> lock(mSubmitMutex);
+		mSubmitExitFlag = true;
+		lock.unlock();
+		mSubmitCondVar.notify_all();
+		mSubmitThread.join();
+		mSubmitThread = {};
+		lock.lock();
+		mSubmitExitFlag = false;
+	}
+}
+
+void VulkanFrameBuffer::SubmitThreadMain()
+{
+	while (true)
+	{
+		std::unique_lock<std::mutex> lock(mSubmitMutex);
+		mSubmitCondVar.wait(lock, [&]() { return mSubmitExitFlag || !mSubmitQueue.empty(); });
+
+		std::vector<VulkanCommandBuffer*> commands;
+		commands.swap(mSubmitQueue);
+		mSubmitItemsActive = commands.size();
+		lock.unlock();
+
+		FlushCommands(commands.data(), commands.size());
+
+		lock.lock();
+		mSubmitItemsActive = 0;
+		if (mSubmitQueue.empty())
+			mSubmitDone.notify_all();
+		if (mSubmitExitFlag)
+			break;
+	}
+}
+
+void VulkanFrameBuffer::FlushCommands(VulkanCommandBuffer **commands, size_t count)
+{
+	int currentIndex = nextSubmitQueue % submitQueueSize;
+
+	if (nextSubmitQueue >= submitQueueSize)
+	{
+		vkWaitForFences(device->device, 1, &mSubmitFence[currentIndex]->fence, VK_TRUE, std::numeric_limits<uint64_t>::max());
+		vkResetFences(device->device, 1, &mSubmitFence[currentIndex]->fence);
+	}
+
+	QueueSubmit submit;
+
+	for (size_t i = 0; i < count; i++)
+		submit.addCommandBuffer(commands[i]);
+
+	if (nextSubmitQueue > 0)
+		submit.addWait(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, mSubmitSemaphore[(nextSubmitQueue - 1) % submitQueueSize].get());
+
+	submit.addSignal(mSubmitSemaphore[currentIndex].get());
+	submit.execute(device, device->graphicsQueue, mSubmitFence[currentIndex].get());
+	nextSubmitQueue++;
+}
+
 void VulkanFrameBuffer::FlushCommands()
 {
 	if (mDrawCommands || mTransferCommands)
 	{
-		int currentIndex = nextSubmitQueue % submitQueueSize;
+		VulkanCommandBuffer *commands[2];
-
+		size_t count = 0;
-		if (nextSubmitQueue >= submitQueueSize)
-		{
-			vkWaitForFences(device->device, 1, &mSubmitFence[currentIndex]->fence, VK_TRUE, std::numeric_limits<uint64_t>::max());
-			vkResetFences(device->device, 1, &mSubmitFence[currentIndex]->fence);
-		}
-
-		QueueSubmit submit;
 
 		if (mTransferCommands)
 		{
 			mTransferCommands->end();
-			submit.addCommandBuffer(mTransferCommands.get());
+			commands[count++] = mTransferCommands.get();
-
 			FrameDeleteList.CommandBuffers.push_back(std::move(mTransferCommands));
 		}
 
 		if (mDrawCommands)
 		{
 			mDrawCommands->end();
-			submit.addCommandBuffer(mDrawCommands.get());
+			commands[count++] = mDrawCommands.get();
-
 			FrameDeleteList.CommandBuffers.push_back(std::move(mDrawCommands));
 		}
 
-		if (nextSubmitQueue > 0)
+		current_rendered_commandbuffers += (int)count;
-			submit.addWait(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, mSubmitSemaphore[(nextSubmitQueue - 1) % submitQueueSize].get());
 
-		submit.addSignal(mSubmitSemaphore[currentIndex].get());
+		if (vk_submit_multithread)
-		submit.execute(device, device->graphicsQueue, mSubmitFence[currentIndex].get());
+		{
-		nextSubmitQueue++;
+			StartSubmitThread();
+			std::unique_lock<std::mutex> lock(mSubmitMutex);
+			for (size_t i = 0; i < count; i++)
+				mSubmitQueue.push_back(commands[i]);
+			lock.unlock();
+			mSubmitCondVar.notify_all();
+		}
+		else
+		{
+			StopSubmitThread();
+			FlushCommands(commands, count);
+		}
 	}
 }
 
@@ -242,6 +318,10 @@ void VulkanFrameBuffer::WaitForCommands(bool finish)
 
 	FlushCommands();
 
+	std::unique_lock<std::mutex> lock(mSubmitMutex);
+	mSubmitDone.wait(lock, [&]() { return mSubmitQueue.empty() && mSubmitItemsActive == 0; });
+	lock.unlock();
+
 	QueueSubmit submit;
 
 	if (nextSubmitQueue > 0)
@@ -280,6 +360,8 @@ void VulkanFrameBuffer::WaitForCommands(bool finish)
 	if (finish)
 	{
 		Finish.Unclock();
+		rendered_commandbuffers = current_rendered_commandbuffers;
+		current_rendered_commandbuffers = 0;
 	}
 }
 

src/rendering/vulkan/system/vk_framebuffer.h

@@ -3,6 +3,9 @@
 #include "gl_sysfb.h"
 #include "vk_device.h"
 #include "vk_objects.h"
+#include <thread>
+#include <mutex>
+#include <condition_variable>
 
 struct FRenderViewpoint;
 class VkSamplerManager;
@@ -111,6 +114,10 @@ private:
 	void CopyScreenToBuffer(int w, int h, void *data);
 	void UpdateShadowMap();
 	void DeleteFrameObjects();
+	void StartSubmitThread();
+	void StopSubmitThread();
+	void SubmitThreadMain();
+	void FlushCommands(VulkanCommandBuffer **commands, size_t count);
 
 	std::unique_ptr<VkShaderManager> mShaderManager;
 	std::unique_ptr<VkSamplerManager> mSamplerManager;
@@ -134,6 +141,14 @@ private:
 	std::unique_ptr<VulkanFence> mRenderFinishedFence;
 
 	VkRenderBuffers *mActiveRenderBuffers = nullptr;
+
+	std::thread mSubmitThread;
+	std::condition_variable mSubmitCondVar;
+	std::condition_variable mSubmitDone;
+	std::mutex mSubmitMutex;
+	std::vector<VulkanCommandBuffer*> mSubmitQueue;
+	size_t mSubmitItemsActive = 0;
+	bool mSubmitExitFlag = false;
 };
 
 inline VulkanFrameBuffer *GetVulkanFrameBuffer() { return static_cast<VulkanFrameBuffer*>(screen); }