[vulkan] Use a command buffer pool manager

Many thanks to Peter and Darian for clearing up my misunderstanding of
how vkResetCommandPool works. The manager creates command buffers from
the command pool on an as-needed basis (when the queue of available
buffers is empty), and keeps track of those buffers in a queue. When the
pool is reset, the queues (one each for primary and secondary command
buffers) are reset such that the tracked buffers are available again.

include/QF/Vulkan/command.h

@@ -7,6 +7,7 @@
 #include <vulkan/vulkan.h>
 
 #include "QF/darray.h"
+#include "QF/qtypes.h"
 
 typedef struct qfv_cmdbufferset_s
 	DARRAY_TYPE (VkCommandBuffer) qfv_cmdbufferset_t;
@@ -29,8 +30,26 @@ typedef struct qfv_bufferimagecopy_s
 #define QFV_AllocBufferImageCopy(num, allocator) \
 	DARRAY_ALLOCFIXED (qfv_bufferimagecopy_t, num, allocator)
 
+typedef struct qfv_cmdpoolmgr_s {
+	qfv_cmdbufferset_t primary;
+	qfv_cmdbufferset_t secondary;
+	size_t      active_primary;
+	size_t      active_secondary;
+	struct qfv_device_s *device;
+	VkCommandPool pool;
+} qfv_cmdpoolmgr_t;
+
 struct qfv_queue_s;
-struct qfv_device_s;
+
+qfv_cmdpoolmgr_t *QFV_CmdPoolManager_Init (qfv_cmdpoolmgr_t *manager,
+										   struct qfv_device_s *device);
+qfv_cmdpoolmgr_t *QFV_CmdPoolManager_New (struct qfv_device_s *device);
+void QFV_CmdPoolManager_Shutdown (qfv_cmdpoolmgr_t *manager);
+void QFV_CmdPoolManager_Delete (qfv_cmdpoolmgr_t *manager);
+void QFV_CmdPoolManager_Reset (qfv_cmdpoolmgr_t *manager);
+VkCommandBuffer QFV_CmdPoolManager_CmdBuffer (qfv_cmdpoolmgr_t *manager,
+											  bool secondary);
+
 VkCommandPool QFV_CreateCommandPool (struct qfv_device_s *device,
 									  uint32_t queueFamily,
 									  int transient, int reset);

include/QF/Vulkan/render.h

@@ -355,14 +355,13 @@ typedef struct qfv_job_s {
 	VkPipelineLayout *layouts;
 	qfv_step_t *steps;
 	qfv_cmdbufferset_t commands;
-	VkCommandPool command_pool;
 } qfv_job_t;
 
 typedef struct qfv_renderframe_s {
 	VkFence     fence;
 	VkSemaphore imageAvailableSemaphore;
 	VkSemaphore renderDoneSemaphore;
-	VkCommandPool command_pool;
+	qfv_cmdpoolmgr_t cmdpool;
 } qfv_renderframe_t;
 
 typedef struct qfv_renderframeset_s
@@ -382,7 +381,7 @@ typedef struct qfv_taskctx_s {
 	VkCommandBuffer cmd;
 } qfv_taskctx_t;
 
-VkCommandBuffer QFV_GetCmdBufffer (struct vulkan_ctx_s *ctx, bool secondary);
+VkCommandBuffer QFV_GetCmdBuffer (struct vulkan_ctx_s *ctx, bool secondary);
 void QFV_AppendCmdBuffer (struct vulkan_ctx_s *ctx, VkCommandBuffer cmd);
 
 void QFV_RunRenderJob (struct vulkan_ctx_s *ctx);

libs/video/renderer/vulkan/command.c

@@ -27,10 +27,94 @@
 #ifdef HAVE_CONFIG_H
 # include "config.h"
 #endif
-
+#include "QF/qtypes.h"
 #include "QF/Vulkan/command.h"
 #include "QF/Vulkan/device.h"
 
+qfv_cmdpoolmgr_t *
+QFV_CmdPoolManager_Init (qfv_cmdpoolmgr_t *manager, qfv_device_t *device)
+{
+	*manager = (qfv_cmdpoolmgr_t) {
+		.primary = DARRAY_STATIC_INIT (16),
+		.secondary = DARRAY_STATIC_INIT (16),
+		.device = device,
+	};
+	auto dfunc = device->funcs;
+
+	VkCommandPoolCreateInfo poolCInfo = {
+		.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
+		.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT,
+		.queueFamilyIndex = device->queue.queueFamily,
+	};
+	dfunc->vkCreateCommandPool (device->dev, &poolCInfo, 0, &manager->pool);
+	return manager;
+}
+
+qfv_cmdpoolmgr_t *
+QFV_CmdPoolManager_New (qfv_device_t *device)
+{
+	return QFV_CmdPoolManager_Init (malloc (sizeof (qfv_cmdpoolmgr_t)), device);
+}
+
+void
+QFV_CmdPoolManager_Shutdown (qfv_cmdpoolmgr_t *manager)
+{
+	auto device = manager->device;
+	auto dfunc = device->funcs;
+	dfunc->vkDestroyCommandPool (device->dev, manager->pool, 0);
+	DARRAY_CLEAR (&manager->primary);
+	DARRAY_CLEAR (&manager->secondary);
+}
+
+void
+QFV_CmdPoolManager_Delete (qfv_cmdpoolmgr_t *manager)
+{
+	QFV_CmdPoolManager_Shutdown (manager);
+	free (manager);
+}
+
+void
+QFV_CmdPoolManager_Reset (qfv_cmdpoolmgr_t *manager)
+{
+	auto device = manager->device;
+	auto dfunc = device->funcs;
+	dfunc->vkResetCommandPool (device->dev, manager->pool, 0);
+	manager->active_primary = 0;
+	manager->active_secondary = 0;
+}
+
+VkCommandBuffer
+QFV_CmdPoolManager_CmdBuffer (qfv_cmdpoolmgr_t *manager, bool secondary)
+{
+	auto device = manager->device;
+	auto dfunc = device->funcs;
+
+	if (secondary) {
+		if (manager->active_secondary < manager->secondary.size) {
+			return manager->secondary.a[manager->active_secondary++];
+		}
+	} else {
+		if (manager->active_primary < manager->primary.size) {
+			return manager->primary.a[manager->active_primary++];
+		}
+	}
+	VkCommandBufferAllocateInfo cinfo = {
+		.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
+		.commandPool = manager->pool,
+		.level = secondary ? VK_COMMAND_BUFFER_LEVEL_SECONDARY
+						   : VK_COMMAND_BUFFER_LEVEL_PRIMARY,
+		.commandBufferCount = 1,
+	};
+	VkCommandBuffer cmd;
+	dfunc->vkAllocateCommandBuffers (device->dev, &cinfo, &cmd);
+	if (secondary) {
+		DARRAY_APPEND (&manager->secondary, cmd);
+	} else {
+		DARRAY_APPEND (&manager->primary, cmd);
+	}
+	return cmd;
+}
+
 VkCommandPool
 QFV_CreateCommandPool (qfv_device_t *device, uint32_t queueFamily,
 					   int transient, int reset)

libs/video/renderer/vulkan/render.c

@@ -54,23 +54,11 @@
 #include "vkparse.h"
 
 VkCommandBuffer
-QFV_GetCmdBufffer (vulkan_ctx_t *ctx, bool secondary)
+QFV_GetCmdBuffer (vulkan_ctx_t *ctx, bool secondary)
 {
-	qfv_device_t *device = ctx->device;
+	auto rctx = ctx->render_context;
-	qfv_devfuncs_t *dfunc = device->funcs;
+	auto rframe = &rctx->frames.a[ctx->curFrame];
-	__auto_type rctx = ctx->render_context;
+	return QFV_CmdPoolManager_CmdBuffer (&rframe->cmdpool, secondary);
-	__auto_type job = rctx->job;
-
-	VkCommandBufferAllocateInfo cinfo = {
-		.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
-		.commandPool = job->command_pool,
-		.level = secondary ? VK_COMMAND_BUFFER_LEVEL_SECONDARY
-						   : VK_COMMAND_BUFFER_LEVEL_PRIMARY,
-		.commandBufferCount = 1,
-	};
-	VkCommandBuffer cmd;
-	dfunc->vkAllocateCommandBuffers (device->dev, &cinfo, &cmd);
-	return cmd;
 }
 
 void
@@ -144,7 +132,7 @@ run_renderpass (qfv_renderpass_t *rp, vulkan_ctx_t *ctx)
 	__auto_type rctx = ctx->render_context;
 	__auto_type job = rctx->job;
 
-	VkCommandBuffer cmd = QFV_GetCmdBufffer (ctx, false);
+	VkCommandBuffer cmd = QFV_GetCmdBuffer (ctx, false);
 	VkCommandBufferBeginInfo beginInfo = {
 		VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
 	};
@@ -154,7 +142,7 @@ run_renderpass (qfv_renderpass_t *rp, vulkan_ctx_t *ctx)
 	dfunc->vkCmdBeginRenderPass (cmd, &rp->beginInfo, rp->subpassContents);
 	for (uint32_t i = 0; i < rp->subpass_count; i++) {
 		__auto_type sp = &rp->subpasses[i];
-		VkCommandBuffer subcmd = QFV_GetCmdBufffer (ctx, true);
+		VkCommandBuffer subcmd = QFV_GetCmdBuffer (ctx, true);
 		run_subpass (sp, subcmd, ctx);
 		dfunc->vkCmdExecuteCommands (cmd, 1, &subcmd);
 		//FIXME comment is a bit off as exactly one buffer is always submitted
@@ -210,7 +198,7 @@ run_compute (qfv_compute_t *comp, vulkan_ctx_t *ctx)
 	__auto_type rctx = ctx->render_context;
 	__auto_type job = rctx->job;
 
-	VkCommandBuffer cmd = QFV_GetCmdBufffer (ctx, false);
+	VkCommandBuffer cmd = QFV_GetCmdBuffer (ctx, false);
 
 	VkCommandBufferBeginInfo beginInfo = {
 		.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
@@ -353,15 +341,13 @@ wait_on_fence (const exprval_t **params, exprval_t *result, exprctx_t *ectx)
 	auto device = ctx->device;
 	auto dfunc = device->funcs;
 	auto dev = device->dev;
-
 	auto rctx = ctx->render_context;
 	auto frame = &rctx->frames.a[ctx->curFrame];
 
 	dfunc->vkWaitForFences (dev, 1, &frame->fence, VK_TRUE, 2000000000);
 
+	QFV_CmdPoolManager_Reset (&frame->cmdpool);
 	auto job = ctx->render_context->job;
-	job->command_pool = frame->command_pool;
-	dfunc->vkResetCommandPool (device->dev, job->command_pool, 0);
 	DARRAY_RESIZE (&job->commands, 0);
 }
 
@@ -457,14 +443,7 @@ QFV_Render_Init (vulkan_ctx_t *ctx)
 							 frame->imageAvailableSemaphore,
 							 va (ctx->va_ctx, "sc image:%zd", i));
 		frame->renderDoneSemaphore = QFV_CreateSemaphore (device);
-		VkCommandPoolCreateInfo poolCInfo = {
+		QFV_CmdPoolManager_Init (&frame->cmdpool, device);
-			.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
-			.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT,
-			.queueFamilyIndex = device->queue.queueFamily,
-		};
-		auto dfunc = device->funcs;
-		dfunc->vkCreateCommandPool (device->dev, &poolCInfo, 0,
-									&frame->command_pool);
 	}
 }
 
@@ -502,7 +481,6 @@ QFV_Render_Shutdown (vulkan_ctx_t *ctx)
 				}
 			}
 		}
-		job->command_pool = 0;
 		DARRAY_CLEAR (&job->commands);
 		free (rctx->job);
 	}
@@ -514,7 +492,7 @@ QFV_Render_Shutdown (vulkan_ctx_t *ctx)
 		df->vkDestroyFence (dev, frame->fence, 0);
 		df->vkDestroySemaphore (dev, frame->imageAvailableSemaphore, 0);
 		df->vkDestroySemaphore (dev, frame->renderDoneSemaphore, 0);
-		df->vkDestroyCommandPool (dev, frame->command_pool, 0);
+		QFV_CmdPoolManager_Shutdown (&frame->cmdpool);
 	}
 	DARRAY_CLEAR (&rctx->frames);
 

libs/video/renderer/vulkan/vulkan_translucent.c

@@ -74,7 +74,7 @@ clear_translucent (const exprval_t **params, exprval_t *result, exprctx_t *ectx)
 	translucentctx_t *tctx = ctx->translucent_context;
 	__auto_type tframe = &tctx->frames.a[ctx->curFrame];
 
-	VkCommandBuffer cmd = QFV_GetCmdBufffer (ctx, false);
+	VkCommandBuffer cmd = QFV_GetCmdBuffer (ctx, false);
 
 	VkCommandBufferInheritanceInfo inherit = {
 		VK_STRUCTURE_TYPE_COMMAND_BUFFER_INHERITANCE_INFO, 0,