From 458da39c395d50039fd2fae2605c3d40d3b4dbc8 Mon Sep 17 00:00:00 2001 From: Magnus Norddahl Date: Fri, 19 Apr 2019 22:42:32 +0200 Subject: [PATCH] - add vk_submit_multithread for doing command buffer submit calls on a worker thread - add vk_submit_size for testing various command buffer sizes before flushing them - add submitted command buffer count to renderstats --- src/rendering/hwrenderer/utility/hw_clock.cpp | 6 +- .../vulkan/renderer/vk_renderstate.cpp | 4 +- .../vulkan/system/vk_framebuffer.cpp | 118 +++++++++++++++--- src/rendering/vulkan/system/vk_framebuffer.h | 15 +++ 4 files changed, 121 insertions(+), 22 deletions(-) diff --git a/src/rendering/hwrenderer/utility/hw_clock.cpp b/src/rendering/hwrenderer/utility/hw_clock.cpp index 67b38ed16..dff667b1a 100644 --- a/src/rendering/hwrenderer/utility/hw_clock.cpp +++ b/src/rendering/hwrenderer/utility/hw_clock.cpp @@ -54,7 +54,7 @@ glcycle_t twoD, Flush3D; glcycle_t MTWait, WTTotal; int vertexcount, flatvertices, flatprimitives; -int rendered_lines,rendered_flats,rendered_sprites,render_vertexsplit,render_texsplit,rendered_decals, rendered_portals; +int rendered_lines,rendered_flats,rendered_sprites,render_vertexsplit,render_texsplit,rendered_decals, rendered_portals, rendered_commandbuffers; int iter_dlightf, iter_dlight, draw_dlight, draw_dlightf; void ResetProfilingData() @@ -113,8 +113,8 @@ static void AppendRenderStats(FString &out) { out.AppendFormat("Walls: %d (%d splits, %d t-splits, %d vertices)\n" "Flats: %d (%d primitives, %d vertices)\n" - "Sprites: %d, Decals=%d, Portals: %d\n", - rendered_lines, render_vertexsplit, render_texsplit, vertexcount, rendered_flats, flatprimitives, flatvertices, rendered_sprites,rendered_decals, rendered_portals ); + "Sprites: %d, Decals=%d, Portals: %d, Command buffers: %d\n", + rendered_lines, render_vertexsplit, render_texsplit, vertexcount, rendered_flats, flatprimitives, flatvertices, rendered_sprites,rendered_decals, rendered_portals, rendered_commandbuffers ); } static void AppendLightStats(FString &out) diff --git a/src/rendering/vulkan/renderer/vk_renderstate.cpp b/src/rendering/vulkan/renderer/vk_renderstate.cpp index 1e6dad299..fd0080c7b 100644 --- a/src/rendering/vulkan/renderer/vk_renderstate.cpp +++ b/src/rendering/vulkan/renderer/vk_renderstate.cpp @@ -17,6 +17,8 @@ #include "hwrenderer/data/hw_viewpointbuffer.h" #include "hwrenderer/data/shaderuniforms.h" +CVAR(Int, vk_submit_size, 1000, 0); + VkRenderState::VkRenderState() { mIdentityMatrix.loadIdentity(); @@ -161,7 +163,7 @@ void VkRenderState::EnableLineSmooth(bool on) void VkRenderState::Apply(int dt) { mApplyCount++; - if (mApplyCount == 1000) + if (mApplyCount >= vk_submit_size) { EndRenderPass(); GetVulkanFrameBuffer()->FlushCommands(); diff --git a/src/rendering/vulkan/system/vk_framebuffer.cpp b/src/rendering/vulkan/system/vk_framebuffer.cpp index eb4b4be70..6cbfa90a0 100644 --- a/src/rendering/vulkan/system/vk_framebuffer.cpp +++ b/src/rendering/vulkan/system/vk_framebuffer.cpp @@ -68,7 +68,11 @@ EXTERN_CVAR(Int, screenblocks) EXTERN_CVAR(Bool, cl_capfps) EXTERN_CVAR(Bool, gl_no_skyclear) +CVAR(Bool, vk_submit_multithread, false, 0) + extern bool NoInterpolateView; +extern int rendered_commandbuffers; +int current_rendered_commandbuffers; VulkanFrameBuffer::VulkanFrameBuffer(void *hMonitor, bool fullscreen, VulkanDevice *dev) : Super(hMonitor, fullscreen) @@ -91,6 +95,8 @@ VulkanFrameBuffer::VulkanFrameBuffer(void *hMonitor, bool fullscreen, VulkanDevi VulkanFrameBuffer::~VulkanFrameBuffer() { + StopSubmitThread(); + // All descriptors must be destroyed before the descriptor pool in renderpass manager is destroyed for (VkHardwareTexture *cur = VkHardwareTexture::First; cur; cur = cur->Next) cur->Reset(); @@ -189,42 +195,112 @@ void VulkanFrameBuffer::DeleteFrameObjects() FrameDeleteList.CommandBuffers.clear(); } +void VulkanFrameBuffer::StartSubmitThread() +{ + if (!mSubmitThread.joinable()) + { + mSubmitThread = std::thread([this] { SubmitThreadMain(); }); + } +} + +void VulkanFrameBuffer::StopSubmitThread() +{ + if (mSubmitThread.joinable()) + { + std::unique_lock lock(mSubmitMutex); + mSubmitExitFlag = true; + lock.unlock(); + mSubmitCondVar.notify_all(); + mSubmitThread.join(); + mSubmitThread = {}; + lock.lock(); + mSubmitExitFlag = false; + } +} + +void VulkanFrameBuffer::SubmitThreadMain() +{ + while (true) + { + std::unique_lock lock(mSubmitMutex); + mSubmitCondVar.wait(lock, [&]() { return mSubmitExitFlag || !mSubmitQueue.empty(); }); + + std::vector commands; + commands.swap(mSubmitQueue); + mSubmitItemsActive = commands.size(); + lock.unlock(); + + FlushCommands(commands.data(), commands.size()); + + lock.lock(); + mSubmitItemsActive = 0; + if (mSubmitQueue.empty()) + mSubmitDone.notify_all(); + if (mSubmitExitFlag) + break; + } +} + +void VulkanFrameBuffer::FlushCommands(VulkanCommandBuffer **commands, size_t count) +{ + int currentIndex = nextSubmitQueue % submitQueueSize; + + if (nextSubmitQueue >= submitQueueSize) + { + vkWaitForFences(device->device, 1, &mSubmitFence[currentIndex]->fence, VK_TRUE, std::numeric_limits::max()); + vkResetFences(device->device, 1, &mSubmitFence[currentIndex]->fence); + } + + QueueSubmit submit; + + for (size_t i = 0; i < count; i++) + submit.addCommandBuffer(commands[i]); + + if (nextSubmitQueue > 0) + submit.addWait(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, mSubmitSemaphore[(nextSubmitQueue - 1) % submitQueueSize].get()); + + submit.addSignal(mSubmitSemaphore[currentIndex].get()); + submit.execute(device, device->graphicsQueue, mSubmitFence[currentIndex].get()); + nextSubmitQueue++; +} + void VulkanFrameBuffer::FlushCommands() { if (mDrawCommands || mTransferCommands) { - int currentIndex = nextSubmitQueue % submitQueueSize; - - if (nextSubmitQueue >= submitQueueSize) - { - vkWaitForFences(device->device, 1, &mSubmitFence[currentIndex]->fence, VK_TRUE, std::numeric_limits::max()); - vkResetFences(device->device, 1, &mSubmitFence[currentIndex]->fence); - } - - QueueSubmit submit; + VulkanCommandBuffer *commands[2]; + size_t count = 0; if (mTransferCommands) { mTransferCommands->end(); - submit.addCommandBuffer(mTransferCommands.get()); - + commands[count++] = mTransferCommands.get(); FrameDeleteList.CommandBuffers.push_back(std::move(mTransferCommands)); } if (mDrawCommands) { mDrawCommands->end(); - submit.addCommandBuffer(mDrawCommands.get()); - + commands[count++] = mDrawCommands.get(); FrameDeleteList.CommandBuffers.push_back(std::move(mDrawCommands)); } - if (nextSubmitQueue > 0) - submit.addWait(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, mSubmitSemaphore[(nextSubmitQueue - 1) % submitQueueSize].get()); + current_rendered_commandbuffers += (int)count; - submit.addSignal(mSubmitSemaphore[currentIndex].get()); - submit.execute(device, device->graphicsQueue, mSubmitFence[currentIndex].get()); - nextSubmitQueue++; + if (vk_submit_multithread) + { + StartSubmitThread(); + std::unique_lock lock(mSubmitMutex); + for (size_t i = 0; i < count; i++) + mSubmitQueue.push_back(commands[i]); + lock.unlock(); + mSubmitCondVar.notify_all(); + } + else + { + StopSubmitThread(); + FlushCommands(commands, count); + } } } @@ -242,6 +318,10 @@ void VulkanFrameBuffer::WaitForCommands(bool finish) FlushCommands(); + std::unique_lock lock(mSubmitMutex); + mSubmitDone.wait(lock, [&]() { return mSubmitQueue.empty() && mSubmitItemsActive == 0; }); + lock.unlock(); + QueueSubmit submit; if (nextSubmitQueue > 0) @@ -280,6 +360,8 @@ void VulkanFrameBuffer::WaitForCommands(bool finish) if (finish) { Finish.Unclock(); + rendered_commandbuffers = current_rendered_commandbuffers; + current_rendered_commandbuffers = 0; } } diff --git a/src/rendering/vulkan/system/vk_framebuffer.h b/src/rendering/vulkan/system/vk_framebuffer.h index 244463c98..bb5946f08 100644 --- a/src/rendering/vulkan/system/vk_framebuffer.h +++ b/src/rendering/vulkan/system/vk_framebuffer.h @@ -3,6 +3,9 @@ #include "gl_sysfb.h" #include "vk_device.h" #include "vk_objects.h" +#include +#include +#include struct FRenderViewpoint; class VkSamplerManager; @@ -111,6 +114,10 @@ private: void CopyScreenToBuffer(int w, int h, void *data); void UpdateShadowMap(); void DeleteFrameObjects(); + void StartSubmitThread(); + void StopSubmitThread(); + void SubmitThreadMain(); + void FlushCommands(VulkanCommandBuffer **commands, size_t count); std::unique_ptr mShaderManager; std::unique_ptr mSamplerManager; @@ -134,6 +141,14 @@ private: std::unique_ptr mRenderFinishedFence; VkRenderBuffers *mActiveRenderBuffers = nullptr; + + std::thread mSubmitThread; + std::condition_variable mSubmitCondVar; + std::condition_variable mSubmitDone; + std::mutex mSubmitMutex; + std::vector mSubmitQueue; + size_t mSubmitItemsActive = 0; + bool mSubmitExitFlag = false; }; inline VulkanFrameBuffer *GetVulkanFrameBuffer() { return static_cast(screen); }