[vulkan] Convert stagebuf to a ring buffer

I should have known this would be necessary, but it fixes the corruption
when updating the scrap.

include/QF/Vulkan/funclist.h

@@ -101,6 +101,7 @@ DEVICE_LEVEL_VULKAN_FUNCTION (vkCreateSemaphore)
 DEVICE_LEVEL_VULKAN_FUNCTION (vkCreateFence)
 DEVICE_LEVEL_VULKAN_FUNCTION (vkWaitForFences)
 DEVICE_LEVEL_VULKAN_FUNCTION (vkResetFences)
+DEVICE_LEVEL_VULKAN_FUNCTION (vkGetFenceStatus)
 DEVICE_LEVEL_VULKAN_FUNCTION (vkQueueSubmit)
 DEVICE_LEVEL_VULKAN_FUNCTION (vkQueueWaitIdle)
 DEVICE_LEVEL_VULKAN_FUNCTION (vkDeviceWaitIdle)

include/QF/Vulkan/staging.h

@@ -1,20 +1,36 @@
 #ifndef __QF_Vulkan_staging_h
 #define __QF_Vulkan_staging_h
 
+typedef struct qfv_packet_s {
+	struct qfv_stagebuf_s *stage;	///< staging buffer that owns this packet
+	VkCommandBuffer cmd;
+	VkFence     fence;
+	size_t      offset;
+	size_t      length;
+} qfv_packet_t;
+
 typedef struct qfv_stagebuf_s {
 	struct qfv_device_s *device;
 	VkBuffer    buffer;
 	VkDeviceMemory memory;
-	size_t      size;
-	size_t      offset;		///< for batching building
+	qfv_packet_t *packet;	///< array of packets for controlling access
+	size_t      num_packets;///< number of packets in array
+	size_t      next_packet;///< index of the next packet to be used
+	size_t      size;		///< actual size of the buffer
+	size_t      end;		///< effective end of the buffer due to early wrap
+	size_t      head;
+	size_t      tail;
 	void       *data;
 } qfv_stagebuf_t;
 
 
 qfv_stagebuf_t *QFV_CreateStagingBuffer (struct qfv_device_s *device,
-										 size_t size);
+										 size_t size, int num_packets,
+										 VkCommandPool cmdPool);
 void QFV_DestroyStagingBuffer (qfv_stagebuf_t *stage);
 void QFV_FlushStagingBuffer (qfv_stagebuf_t *stage, size_t offset, size_t size);
-void *QFV_ClaimStagingBuffer (qfv_stagebuf_t *stage, size_t size);
+qfv_packet_t *QFV_PacketAcquire (qfv_stagebuf_t *stage);
+void *QFV_PacketExtend (qfv_packet_t *packet, size_t size);
+void QFV_PacketSubmit (qfv_packet_t *packet);
 
 #endif//__QF_Vulkan_staging_h

include/QF/Vulkan/texture.h

@@ -18,26 +18,8 @@ VkImageView QFV_ScrapImageView (scrap_t *scrap) __attribute__((pure));
 subpic_t *QFV_ScrapSubpic (scrap_t *scrap, int width, int height);
 void QFV_SubpicDelete (subpic_t *subpic);
 
-/** Add an update region to the batch queue.
- *
- * The region to be updated is recorded in the batch queue, space allocated
- * in the staging buffer, and a pointer to the allocated space is returned.
- *
- * \note No data is stransfered. This facilitates writing generated data
- * directly to the staging buffer.
- */
 void *QFV_SubpicBatch (subpic_t *subpic, struct qfv_stagebuf_s *stage);
 
-/** Flush all batched subpic updates.
- *
- * The offset in the staging bufffer \a stage is reset to 0. The command
- * buffer is populated with the appropriate image layout barriers and the
- * necessary copy commands.
- *
- * \note The command buffer is neither begun nor ended, nor is it submitted
- * to a queue. This is to maximize flexibility in command buffer usage.
- */
-void QFV_ScrapFlush (scrap_t *scrap, struct qfv_stagebuf_s *stage,
-					 VkCommandBuffer cmd);
+void QFV_ScrapFlush (scrap_t *scrap);
 
 #endif//__QF_Vulkan_texture_h

libs/video/renderer/Makemodule.am

@@ -1,3 +1,4 @@
+include libs/video/renderer/vulkan/test/Makemodule.am
 include libs/video/renderer/vulkan/vkgen/Makemodule.am
 
 #lib_LTLIBRARIES += @VID_REND_TARGETS@

libs/video/renderer/vulkan/staging.c

@@ -47,6 +47,7 @@
 #include "QF/sys.h"
 #include "QF/Vulkan/qf_vid.h"
 #include "QF/Vulkan/buffer.h"
+#include "QF/Vulkan/command.h"
 #include "QF/Vulkan/device.h"
 #include "QF/Vulkan/instance.h"
 #include "QF/Vulkan/staging.h"
@@ -54,21 +55,39 @@
 #include "vid_vulkan.h"
 
 qfv_stagebuf_t *
-QFV_CreateStagingBuffer (qfv_device_t *device, size_t size)
+QFV_CreateStagingBuffer (qfv_device_t *device, size_t size, int num_packets,
+						 VkCommandPool cmdPool)
 {
 	qfv_devfuncs_t *dfunc = device->funcs;
 
-	qfv_stagebuf_t *stage = malloc (sizeof (qfv_stagebuf_t));
-	stage->size = size;
-	stage->offset = 0;
+	qfv_stagebuf_t *stage = malloc (sizeof (qfv_stagebuf_t)
+									+ num_packets * sizeof (qfv_packet_t));
 	stage->device = device;
 	stage->buffer = QFV_CreateBuffer (device, size,
 									  VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
 	stage->memory = QFV_AllocBufferMemory (device, stage->buffer,
 										   VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT,
 										   size, 0);
-	QFV_BindBufferMemory (device, stage->buffer, stage->memory, 0);
+	stage->packet = (qfv_packet_t *) (stage + 1);
+	stage->num_packets = num_packets;
+	stage->next_packet = 0;
+	stage->size = size;
+	stage->end = size;
+	stage->head = 0;
+	stage->tail = 0;
+
 	dfunc->vkMapMemory (device->dev, stage->memory, 0, size, 0, &stage->data);
+	QFV_BindBufferMemory (device, stage->buffer, stage->memory, 0);
+
+	__auto_type bufferset = QFV_AllocCommandBufferSet (num_packets, alloca);
+	QFV_AllocateCommandBuffers (device, cmdPool, 0, bufferset);
+	for (int i = 0; i < num_packets; i++) {
+		stage->packet[i].stage = stage;
+		stage->packet[i].cmd = bufferset->a[i];
+		stage->packet[i].fence = QFV_CreateFence (device, 1);
+		stage->packet[i].offset = 0;
+		stage->packet[i].length = 0;
+	}
 	return stage;
 }
 
@@ -78,6 +97,10 @@ QFV_DestroyStagingBuffer (qfv_stagebuf_t *stage)
 	qfv_device_t *device = stage->device;
 	qfv_devfuncs_t *dfunc = device->funcs;
 
+	for (size_t i = 0; i < stage->num_packets; i++) {
+		dfunc->vkDestroyFence (device->dev, stage->packet[i].fence, 0);
+	}
+
 	dfunc->vkUnmapMemory (device->dev, stage->memory);
 	dfunc->vkFreeMemory (device->dev, stage->memory, 0);
 	dfunc->vkDestroyBuffer (device->dev, stage->buffer, 0);
@@ -100,13 +123,150 @@ QFV_FlushStagingBuffer (qfv_stagebuf_t *stage, size_t offset, size_t size)
 	dfunc->vkFlushMappedMemoryRanges (device->dev, 1, &range);
 }
 
-void *
-QFV_ClaimStagingBuffer (qfv_stagebuf_t *stage, size_t size)
+static void
+advance_tail (qfv_stagebuf_t *stage, qfv_packet_t *packet)
 {
-	if (stage->offset + size > stage->size) {
+	qfv_device_t *device = stage->device;
+	qfv_devfuncs_t *dfunc = device->funcs;
+
+	qfv_packet_t *start = packet;
+
+	while (1) {
+		if ((size_t) (++packet - stage->packet) >= stage->num_packets) {
+			packet = stage->packet;
+		}
+		if (packet != start
+			&& (dfunc->vkGetFenceStatus (device->dev, packet->fence)
+				== VK_SUCCESS)) {
+			if (packet->length == 0) {
+				continue;
+			}
+			if ((stage->tail == stage->end && packet->offset == 0)
+				|| stage->tail == packet->offset) {
+				stage->tail = packet->offset + packet->length;
+				packet->length = 0;
+				if (stage->tail >= stage->end) {
+					stage->end = stage->size;
+					stage->tail = stage->size;
+				}
+			}
+			continue;
+		}
+		// Packets are always aquired in sequence and thus the first busy
+		// packet after the start packet marks the end of available space.
+		// Alternatively, there is only one packet and we've looped around
+		// back to the start packet. Must ensure the tail is updated
+		if (stage->tail >= stage->end && packet->offset == 0) {
+			stage->end = stage->size;
+			stage->tail = packet->offset;
+		}
+		break;
+	}
+}
+
+qfv_packet_t *
+QFV_PacketAcquire (qfv_stagebuf_t *stage)
+{
+	qfv_device_t *device = stage->device;
+	qfv_devfuncs_t *dfunc = device->funcs;
+	qfv_packet_t *packet = &stage->packet[stage->next_packet++];
+	stage->next_packet %= stage->num_packets;
+
+	dfunc->vkWaitForFences (device->dev, 1, &packet->fence, VK_TRUE, ~0ull);
+
+	advance_tail (stage, packet);
+	if (stage->head == stage->size) {
+		stage->head = 0;
+	}
+	packet->offset = stage->head;
+	packet->length = 0;
+
+	dfunc->vkResetFences (device->dev, 1, &packet->fence);
+	dfunc->vkResetCommandBuffer (packet->cmd, 0);
+	VkCommandBufferBeginInfo beginInfo = {
+		VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, 0,
+		VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, 0,
+	};
+	dfunc->vkBeginCommandBuffer (packet->cmd, &beginInfo);
+
+	return packet;
+}
+
+void *
+QFV_PacketExtend (qfv_packet_t *packet, size_t size)
+{
+	qfv_stagebuf_t *stage = packet->stage;
+	if (!size || size > stage->size) {
 		return 0;
 	}
-	void       *data = (byte *) stage->data + stage->offset;
-	stage->offset += (size + 3) & ~3;
+
+	//FIXME extra logic may be needed to wait wait for space to become
+	//available when the requested size should fit but can't due to in-flight
+	//packets
+	advance_tail (stage, packet);
+
+	size_t      head = stage->head;
+	size_t      end = stage->end;
+	if (head + size > stage->end) {
+		if (packet->length) {
+			// packets cannot wrap around the buffer (must use separate
+			// packets)
+			return 0;
+		}
+		if (stage->tail == 0) {
+			// the beginning of the the staging buffer is occupied
+			return 0;
+		}
+		packet->offset = 0;
+		head = 0;
+		end = stage->head;
+	}
+	if (head < stage->tail && head + size > stage->tail) {
+		// not enough room for the sub-packet
+		return 0;
+	}
+	void       *data = (byte *) stage->data + head;
+	stage->end = end;
+	stage->head = head + size;
+	packet->length += size;
 	return data;
 }
+
+void
+QFV_PacketSubmit (qfv_packet_t *packet)
+{
+	qfv_stagebuf_t *stage = packet->stage;
+	qfv_device_t *device = stage->device;
+	qfv_devfuncs_t *dfunc = device->funcs;
+
+	if (!packet->length) {
+		// XXX at this stage, this looks the same as below, I think a queue
+		// completing is the only way to set a fence (other than creation),
+		// so submit the (hopefully) empty command buffer so the fence becomes
+		// set, but without waiting on or triggering any semaphores.
+		dfunc->vkEndCommandBuffer (packet->cmd);
+		VkSubmitInfo submitInfo = {
+			VK_STRUCTURE_TYPE_SUBMIT_INFO, 0,
+			0, 0, 0,
+			1, &packet->cmd,
+			0, 0,
+		};
+		dfunc->vkQueueSubmit (device->queue.queue, 1, &submitInfo,
+							  packet->fence);
+		return;
+	}
+
+	QFV_FlushStagingBuffer (stage, packet->offset, packet->length);
+
+	dfunc->vkEndCommandBuffer (packet->cmd);
+	//XXX it may become necessary to pass in semaphores etc (maybe add to
+	//packet?)
+	VkSubmitInfo submitInfo = {
+		VK_STRUCTURE_TYPE_SUBMIT_INFO, 0,
+		0, 0, 0,
+		1, &packet->cmd,
+		0, 0,
+	};
+	// The fence was reset when the packet was acquired
+	dfunc->vkQueueSubmit (device->queue.queue, 1, &submitInfo, packet->fence);
+}

libs/video/renderer/vulkan/test/Makemodule.am

@@ -0,0 +1,19 @@
+libs_video_renderer_vulkan_tests = \
+	libs/video/renderer/vulkan/test/test-staging \
+	$e
+
+TESTS += $(libs_video_renderer_vulkan_tests)
+
+check_PROGRAMS += $(libs_video_renderer_vulkan_tests)
+
+libs_video_renderer_vulkan_test_libs= \
+	libs/video/renderer/vid_render_vulkan.la \
+	libs/util/libQFutil.la
+
+libs_video_renderer_vulkan_test_test_staging_SOURCES= 		\
+	libs/video/renderer/vulkan/test/test-staging.c	\
+	$e
+libs_video_renderer_vulkan_test_test_staging_LDADD=			\
+	$(libs_video_renderer_vulkan_test_libs)
+libs_video_renderer_vulkan_test_test_staging_DEPENDENCIES=	\
+	$(libs_video_renderer_vulkan_test_libs)

libs/video/renderer/vulkan/test/test-staging.c

@@ -0,0 +1,438 @@
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "QF/Vulkan/qf_vid.h"
+#include "QF/Vulkan/device.h"
+#include "QF/Vulkan/instance.h"
+#include "QF/Vulkan/staging.h"
+
+void *stage_memory;
+
+static VkResult
+vkMapMemory (VkDevice device, VkDeviceMemory memory,
+			 VkDeviceSize offset, VkDeviceSize size, VkMemoryMapFlags flags,
+			 void **data)
+{
+	char       *buf = calloc (1, size);
+	stage_memory = buf;
+	*data = buf;
+	return VK_SUCCESS;
+}
+
+static VkResult
+vkBindBufferMemory (VkDevice device, VkBuffer buffer, VkDeviceMemory memory,
+					VkDeviceSize offset)
+{
+	return VK_SUCCESS;
+}
+
+static VkResult
+vkCreateBuffer (VkDevice device, const VkBufferCreateInfo *cinfo,
+				const VkAllocationCallbacks *allocator, VkBuffer *buffer)
+{
+	*buffer = 0;
+	return VK_SUCCESS;
+}
+
+static void
+vkGetBufferMemoryRequirements (VkDevice device, VkBuffer buffer,
+							   VkMemoryRequirements *requirements)
+{
+	memset (requirements, 0, sizeof (*requirements));
+}
+
+static VkResult
+vkAllocateCommandBuffers (VkDevice device,
+						  const VkCommandBufferAllocateInfo *info,
+						  VkCommandBuffer *buffers)
+{
+	static size_t cmdBuffer = 0;
+
+	for (uint32_t i = 0; i < info->commandBufferCount; i++) {
+		buffers[i] = (VkCommandBuffer) ++cmdBuffer;
+	}
+	return VK_SUCCESS;
+}
+
+static VkResult
+vkCreateFence (VkDevice device, const VkFenceCreateInfo *info,
+			   const VkAllocationCallbacks *allocator, VkFence *fence)
+{
+	int       *f = malloc (sizeof (int));
+	*f = info->flags & VK_FENCE_CREATE_SIGNALED_BIT ? 1 : 0;
+	*(int **)fence = f;
+	return VK_SUCCESS;
+}
+
+static VkResult
+vkWaitForFences (VkDevice device, uint32_t fenceCount, const VkFence *fences,
+				 VkBool32 waitAll, uint64_t timeout)
+{
+	for (uint32_t i = 0; i < fenceCount; i++) {
+		int        *f = (int *)fences[i];
+		*f = 0;
+	}
+	return VK_SUCCESS;
+}
+
+static VkResult
+vkResetFences (VkDevice device, uint32_t fenceCount, const VkFence *fences)
+{
+	for (uint32_t i = 0; i < fenceCount; i++) {
+		int        *f = (int *)fences[i];
+		*f = 0;
+	}
+	return VK_SUCCESS;
+}
+
+static VkResult
+vkGetFenceStatus (VkDevice device, VkFence fence)
+{
+	int        *f = (int *)fence;
+	return *f ? VK_SUCCESS : VK_NOT_READY;
+}
+
+static VkResult
+vkResetCommandBuffer (VkCommandBuffer buffer, VkCommandBufferResetFlags flags)
+{
+	return VK_SUCCESS;
+}
+
+static VkResult
+vkBeginCommandBuffer (VkCommandBuffer buffer,
+					  const VkCommandBufferBeginInfo *info)
+{
+	return VK_SUCCESS;
+}
+
+static VkResult
+vkEndCommandBuffer (VkCommandBuffer buffer)
+{
+	return VK_SUCCESS;
+}
+
+static VkResult
+vkFlushMappedMemoryRanges (VkDevice device, uint32_t count,
+						   const VkMappedMemoryRange *ranges)
+{
+	return VK_SUCCESS;
+}
+
+static VkResult
+vkQueueSubmit (VkQueue queue, uint32_t count, const VkSubmitInfo *submits,
+			   VkFence fence)
+{
+	int        *f = (int *)fence;
+	*f = 1;
+	return VK_SUCCESS;
+}
+
+qfv_devfuncs_t dfuncs = {
+	vkCreateBuffer:vkCreateBuffer,
+	vkGetBufferMemoryRequirements:vkGetBufferMemoryRequirements,
+	vkMapMemory:vkMapMemory,
+	vkBindBufferMemory:vkBindBufferMemory,
+	vkAllocateCommandBuffers:vkAllocateCommandBuffers,
+	vkCreateFence:vkCreateFence,
+	vkWaitForFences:vkWaitForFences,
+	vkResetFences:vkResetFences,
+	vkGetFenceStatus:vkGetFenceStatus,
+	vkResetCommandBuffer:vkResetCommandBuffer,
+	vkBeginCommandBuffer:vkBeginCommandBuffer,
+	vkEndCommandBuffer:vkEndCommandBuffer,
+	vkFlushMappedMemoryRanges:vkFlushMappedMemoryRanges,
+	vkQueueSubmit:vkQueueSubmit,
+};
+qfv_physdev_t physDev;
+qfv_device_t device = {
+	physDev:&physDev,
+	funcs:&dfuncs,
+};
+
+static void __attribute__ ((format (printf, 2, 3), noreturn))
+_error (int line, const char *fmt, ...)
+{
+	va_list     args;
+
+	va_start (args, fmt);
+	fprintf (stderr, "%d: ", line);
+	vfprintf (stderr, fmt, args);
+	fprintf (stderr, "\n");
+	va_end (args);
+	exit(1);
+}
+#define error(fmt...) _error(__LINE__, fmt)
+
+int
+main (void)
+{
+	qfv_stagebuf_t *stage = QFV_CreateStagingBuffer (&device, 1024, 4, 0);
+
+	if (stage->num_packets != 4) {
+		error ("stage has incorrect packet count: %zd\n", stage->num_packets);
+	}
+	if (stage->next_packet != 0) {
+		error ("stage has incorrect next_packet: %zd\n", stage->next_packet);
+	}
+	if (stage->size != 1024) {
+		error ("stage has incorrect size: %zd\n", stage->size);
+	}
+	if (stage->end != stage->size) {
+		error ("stage has incorrect end: %zd\n", stage->end);
+	}
+	if (stage->head || stage->tail != stage->head) {
+		error ("stage ring buffer not initialized: h:%zd t:%zd\n",
+			   stage->head, stage->tail);
+	}
+	if (!stage->data || stage->data != stage_memory) {
+		error ("stage memory not mapped: d:%p, m:%p\n",
+			   stage->data, stage_memory);
+	}
+
+	for (size_t i = 0; i < stage->num_packets; i++) {
+		qfv_packet_t *p = &stage->packet[i];
+		if (p->stage != stage) {
+			error ("packet[%zd] stage not set: ps:%p s:%p\n", i,
+				   p->stage, stage);
+		}
+		if (!p->cmd) {
+			error ("packet[%zd] has no command buffer\n", i);
+		}
+		if (!p->fence) {
+			error ("packet[%zd] has no fence\n", i);
+		}
+		for (size_t j = 0; j < i; j++) {
+			qfv_packet_t *q = &stage->packet[j];
+			if (q->cmd == p->cmd || q->fence == p->fence) {
+				error ("packet[%zd] has dup fence or cmd buf\n", i);
+			}
+		}
+		if (vkGetFenceStatus (device.dev, p->fence) != VK_SUCCESS) {
+			error ("packet[%zd].fence is not signaled\n", i);
+		}
+		if (p->offset || p->length) {
+			error ("packet[%zd] size/length not initialized: o:%zd l:%zd\n",
+				   i, p->offset, p->length);
+		}
+	}
+
+	qfv_packet_t *packet = QFV_PacketAcquire (stage);
+	if (vkGetFenceStatus (device.dev, packet->fence) != VK_NOT_READY) {
+		error ("packet.fence is signaled\n");
+	}
+
+	void *data;
+	size_t old_head, old_tail;
+
+	old_head = stage->head;
+	old_tail = stage->tail;
+	data = QFV_PacketExtend (packet, 0);
+	if (data) {
+		error ("0 byte extend did not return null\n");
+	}
+	if (stage->head != old_head || stage->tail != old_tail) {
+		error ("0 byte extend moved head or tail\n");
+	}
+
+	data = QFV_PacketExtend (packet, 2048);
+	if (data) {
+		error ("2048 byte extend did not return null\n");
+	}
+	if (stage->head != old_head || stage->tail != old_tail) {
+		error ("2048 byte extend moved head or tail\n");
+	}
+
+	data = QFV_PacketExtend (packet, 1024);
+	if (!data) {
+		error ("1024 byte extend failed\n");
+	}
+	if (stage->head == old_head) {
+		error ("1024 byte extend did not move head\n");
+	}
+	if (stage->tail != old_tail) {
+		error ("1024 byte extend moved tail\n");
+	}
+	if (stage->head > stage->size) {
+		error ("stage head out of bounds: %zd\n", stage->head);
+	}
+	if (packet->offset != old_head || packet->length != 1024) {
+		error ("packet offset/size incorrect: p:%zd,%zd h:%zd\n",
+			   packet->offset, packet->length, old_head);
+	}
+	if (stage->head < packet->offset + packet->length) {
+		error ("stage head before end of packet: %zd, pe:%zd\n",
+			   stage->head, packet->offset + packet->length);
+	}
+
+	old_head = stage->head;
+	old_tail = stage->tail;
+	data = QFV_PacketExtend (packet, 16);
+	if (data) {
+		error ("16 byte extend in full stage did not return null\n");
+	}
+	if (stage->head != old_head || stage->tail != old_tail) {
+		error ("16 byte extend moved head or tail\n");
+	}
+	QFV_PacketSubmit (packet);
+	if (vkGetFenceStatus (device.dev, packet->fence) != VK_SUCCESS) {
+		error ("packet.fence is not signaled\n");
+	}
+
+	if (stage->head != 1024 || stage->tail != 0) {
+		error ("stage head or tail not as expected: h: %zd t:%zd\n",
+			   stage->head, stage->tail);
+	}
+
+	qfv_packet_t *packet2 = QFV_PacketAcquire (stage);
+	if (!packet2 || packet2 == packet) {
+		error ("did not get new packet: n:%p o:%p\n", packet2, packet);
+	}
+	packet = packet2;
+	if (packet->offset != 0 || stage->head != 0 || stage->tail != 0) {
+		error ("new packet did not wrap: p:%zd h:%zd t:%zd\n",
+			   packet2->offset, stage->head, stage->tail);
+	}
+
+	old_head = stage->head;
+	old_tail = stage->tail;
+	data = QFV_PacketExtend (packet, 768);
+	if (!data) {
+		error ("768 byte extend failed\n");
+	}
+	if (stage->head == old_head) {
+		error ("768 byte extend did not move head\n");
+	}
+	if (stage->tail != 0) {
+		error ("768 byte extend dit not wrap tail: %zd\n", stage->tail);
+	}
+	if (stage->head > stage->size) {
+		error ("stage head out of bounds: %zd\n", stage->head);
+	}
+	if (packet->offset != old_head || packet->length != 768) {
+		error ("packet offset/size incorrect: p:%zd,%zd h:%zd\n",
+			   packet->offset, packet->length, old_head);
+	}
+	if (stage->head < packet->offset + packet->length) {
+		error ("stage head before end of packet: %zd, pe:%zd\n",
+			   stage->head, packet->offset + packet->length);
+	}
+
+	// test attempting to wrap the packet (without needed space)
+	old_head = stage->head;
+	old_tail = stage->tail;
+	data = QFV_PacketExtend (packet, 512);
+	if (data) {
+		error ("512 byte extend in partially full stage succeeded\n");
+	}
+	if (stage->head != old_head || stage->tail != old_tail) {
+		error ("512 byte extend moved head or tail\n");
+	}
+	QFV_PacketSubmit (packet);
+
+	if (stage->head != 768 || stage->tail != 0) {
+		error ("stage head or tail not as expected: h: %zd t:%zd\n",
+			   stage->head, stage->tail);
+	}
+
+	packet = QFV_PacketAcquire (stage);
+
+	// test wrapping a new packet
+	data = QFV_PacketExtend (packet, 512);
+	if (!data) {
+		error ("512 byte extend failed\n");
+	}
+	if (packet->offset != 0 || packet->length != 512) {
+		error ("packet offset/size incorrect: p:%zd,%zd\n",
+			   packet->offset, packet->length);
+	}
+	if (stage->head != 512 || stage->tail != stage->end || stage->end !=768) {
+		error ("stage head or tail not as expected: h: %zd t:%zd\n",
+			   stage->head, stage->tail);
+	}
+	data = QFV_PacketExtend (packet, 512);
+	if (!data) {
+		error ("second 512 byte extend failed\n");
+	}
+	if (packet->offset != 0 || packet->length != 1024) {
+		error ("packet offset/size incorrect: p:%zd,%zd\n",
+			   packet->offset, packet->length);
+	}
+	if (stage->head != 1024 || stage->tail != 0 || stage->end != 1024) {
+		error ("stage head or tail not as expected: h: %zd t:%zd\n",
+			   stage->head, stage->tail);
+	}
+	QFV_PacketSubmit (packet);
+
+	packet = QFV_PacketAcquire (stage);
+	data = QFV_PacketExtend (packet, 512);
+	if (!data) {
+		error ("512 byte extend failed\n");
+	}
+	if (packet->offset != 0 || packet->length != 512) {
+		error ("packet offset/size incorrect: p:%zd,%zd\n",
+			   packet->offset, packet->length);
+	}
+	QFV_PacketSubmit (packet);
+
+	if (stage->head != 512 || stage->tail != 0 || stage->end != 1024) {
+		error ("stage head or tail not as expected: h: %zd t:%zd\n",
+			   stage->head, stage->tail);
+	}
+
+	packet = QFV_PacketAcquire (stage);
+	data = QFV_PacketExtend (packet, 256);
+	if (!data) {
+		error ("256 byte extend failed\n");
+	}
+	if (packet->offset != 512 || packet->length != 256) {
+		error ("packet offset/size incorrect: p:%zd,%zd\n",
+			   packet->offset, packet->length);
+	}
+	if (stage->head != 768 || stage->tail != 512 || stage->end != 1024) {
+		error ("stage head or tail not as expected: h: %zd t:%zd\n",
+			   stage->head, stage->tail);
+	}
+	// don't submit yet. Normally, it would be an error, but the test harness
+	// needs to keep the packet on hand for the following tests to work
+	packet2 = QFV_PacketAcquire (stage);
+	old_head = stage->head;
+	old_tail = stage->tail;
+	data = QFV_PacketExtend (packet2, 768);
+	if (data) {
+		error ("768 byte extend did not return null\n");
+	}
+	if (stage->head != old_head || stage->tail != old_tail) {
+		error ("768 byte extend moved head or tail\n");
+	}
+
+	//should wrap
+	data = QFV_PacketExtend (packet2, 512);
+	if (!data) {
+		error ("512 byte extend failed\n");
+	}
+	if (packet2->offset != 0 || packet2->length != 512) {
+		error ("packet offset/size incorrect: p:%zd,%zd\n",
+			   packet2->offset, packet2->length);
+	}
+	if (stage->head != 512 || stage->tail != 512 || stage->end != 768) {
+		error ("stage head or tail not as expected: h: %zd t:%zd\n",
+			   stage->head, stage->tail);
+	}
+
+	//submit the first packet
+	QFV_PacketSubmit (packet);
+
+	packet = QFV_PacketAcquire (stage);
+	old_head = stage->head;
+	old_tail = stage->tail;
+	data = QFV_PacketExtend (packet, 768);
+	if (data) {
+		error ("768 byte extend did not return null\n");
+	}
+	if (stage->head != old_head || stage->tail != old_tail) {
+		error ("768 byte extend moved head or tail\n");
+	}
+	return 0;
+}

libs/video/renderer/vulkan/texture.c

@@ -65,6 +65,7 @@ struct scrap_s {
 	VkDeviceMemory memory;
 	VkImageView view;
 	size_t      bpp;
+	qfv_packet_t *packet;
 	vrect_t    *batch;
 	vrect_t   **batch_tail;
 	vrect_t    *batch_free;
@@ -119,6 +120,7 @@ QFV_CreateScrap (qfv_device_t *device, int size, QFVFormat format)
 	scrap->bpp = bpp;
 	scrap->subpics = 0;
 	scrap->device = device;
+	scrap->packet = 0;
 	scrap->batch = 0;
 	scrap->batch_tail = &scrap->batch;
 	scrap->batch_free = 0;
@@ -215,9 +217,21 @@ QFV_SubpicBatch (subpic_t *subpic, qfv_stagebuf_t *stage)
 	byte       *dest;
 	size_t      size;
 
-	size = subpic->width * subpic->height * scrap->bpp;
-	if (!(dest = QFV_ClaimStagingBuffer (stage, size))) {
-		return 0;
+	if (!scrap->packet) {
+		scrap->packet = QFV_PacketAcquire (stage);
+	}
+	size = (subpic->width * subpic->height * scrap->bpp + 3) & ~3;
+	if (!(dest = QFV_PacketExtend (scrap->packet, size))) {
+		if (scrap->packet->length) {
+			QFV_ScrapFlush (scrap);
+
+			scrap->packet = QFV_PacketAcquire (stage);
+			dest = QFV_PacketExtend (scrap->packet, size);
+		}
+		if (!dest) {
+			printf ("could not get space for update\n");
+			return 0;
+		}
 	}
 
 	if (scrap->batch_free) {
@@ -238,7 +252,7 @@ QFV_SubpicBatch (subpic_t *subpic, qfv_stagebuf_t *stage)
 }
 
 void
-QFV_ScrapFlush (scrap_t *scrap, qfv_stagebuf_t *stage, VkCommandBuffer cmd)
+QFV_ScrapFlush (scrap_t *scrap)
 {
 	qfv_device_t *device = scrap->device;
 	qfv_devfuncs_t *dfunc = device->funcs;
@@ -247,6 +261,8 @@ QFV_ScrapFlush (scrap_t *scrap, qfv_stagebuf_t *stage, VkCommandBuffer cmd)
 		return;
 	}
 
+	qfv_packet_t *packet = scrap->packet;
+	qfv_stagebuf_t *stage = packet->stage;
 	size_t      i;
 	__auto_type copy = QFV_AllocBufferImageCopy (128, alloca);
 	memset (copy->a, 0, 128 * sizeof (copy->a[0]));
@@ -257,7 +273,6 @@ QFV_ScrapFlush (scrap_t *scrap, qfv_stagebuf_t *stage, VkCommandBuffer cmd)
 		copy->a[i].imageExtent.depth = 1;
 	}
 
-	QFV_FlushStagingBuffer (stage, 0, stage->offset);
 
 	VkImageMemoryBarrier barrier;
 	qfv_pipelinestagepair_t stages;
@@ -265,10 +280,11 @@ QFV_ScrapFlush (scrap_t *scrap, qfv_stagebuf_t *stage, VkCommandBuffer cmd)
 	stages = imageLayoutTransitionStages[qfv_LT_Undefined_to_TransferDst];
 	barrier = imageLayoutTransitionBarriers[qfv_LT_Undefined_to_TransferDst];
 	barrier.image = scrap->image;
-	dfunc->vkCmdPipelineBarrier (cmd, stages.src, stages.dst, 0, 0, 0, 0, 0,
+	dfunc->vkCmdPipelineBarrier (packet->cmd, stages.src, stages.dst,
+								 0, 0, 0, 0, 0,
 								 1, &barrier);
 
-	size_t      offset = 0, size;
+	size_t      offset = packet->offset, size;
 	vrect_t    *batch = scrap->batch;
 	while (scrap->batch_count) {
 		for (i = 0; i < scrap->batch_count && i < 128; i++) {
@@ -283,7 +299,7 @@ QFV_ScrapFlush (scrap_t *scrap, qfv_stagebuf_t *stage, VkCommandBuffer cmd)
 			offset += (size + 3) & ~3;
 			batch = batch->next;
 		}
-		dfunc->vkCmdCopyBufferToImage (cmd, stage->buffer, scrap->image,
+		dfunc->vkCmdCopyBufferToImage (packet->cmd, stage->buffer, scrap->image,
 									   VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
 									   i, copy->a);
 		scrap->batch_count -= i;
@@ -292,12 +308,15 @@ QFV_ScrapFlush (scrap_t *scrap, qfv_stagebuf_t *stage, VkCommandBuffer cmd)
 	stages = imageLayoutTransitionStages[qfv_LT_TransferDst_to_ShaderReadOnly];
 	barrier=imageLayoutTransitionBarriers[qfv_LT_TransferDst_to_ShaderReadOnly];
 	barrier.image = scrap->image;
-	dfunc->vkCmdPipelineBarrier (cmd, stages.src, stages.dst, 0, 0, 0, 0, 0,
+	dfunc->vkCmdPipelineBarrier (packet->cmd, stages.src, stages.dst,
+								 0, 0, 0, 0, 0,
 								 1, &barrier);
 
 	*scrap->batch_tail = scrap->batch_free;
 	scrap->batch_free = scrap->batch;
 	scrap->batch = 0;
 	scrap->batch_tail = &scrap->batch;
-	stage->offset = 0;
+
+	QFV_PacketSubmit (scrap->packet);
+	scrap->packet = 0;
 }

libs/video/renderer/vulkan/vulkan_draw.c

@@ -158,29 +158,7 @@ destroy_quad_buffers (vulkan_ctx_t *ctx)
 static void
 flush_draw_scrap (vulkan_ctx_t *ctx)
 {
-	qfv_device_t *device = ctx->device;
-	qfv_devfuncs_t *dfunc = device->funcs;
-	VkCommandBuffer cmd = ctx->cmdbuffer;
-
-	dfunc->vkWaitForFences (device->dev, 1, &ctx->fence, VK_TRUE, ~0ull);
-	dfunc->vkResetCommandBuffer (cmd, 0);
-	VkCommandBufferBeginInfo beginInfo = {
-		VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, 0,
-		VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, 0,
-	};
-	dfunc->vkBeginCommandBuffer (cmd, &beginInfo);
-	QFV_ScrapFlush (draw_scrap, draw_stage, cmd);
-	dfunc->vkEndCommandBuffer (cmd);
-
-	VkSubmitInfo submitInfo = {
-		VK_STRUCTURE_TYPE_SUBMIT_INFO, 0,
-		0, 0, 0,
-		1, &cmd,
-		0, 0,
-	};
-	dfunc->vkResetFences (device->dev, 1, &ctx->fence);
-	dfunc->vkQueueSubmit (device->queue.queue, 1, &submitInfo, ctx->fence);
-	dfunc->vkWaitForFences (device->dev, 1, &ctx->fence, VK_TRUE, ~0ull);//FIXME
+	QFV_ScrapFlush (draw_scrap);
 }
 
 static void
@@ -326,7 +304,8 @@ Vulkan_Draw_Init (vulkan_ctx_t *ctx)
 
 	create_quad_buffers (ctx);
 	draw_scrap = QFV_CreateScrap (device, 2048, QFV_RGBA);
-	draw_stage = QFV_CreateStagingBuffer (device, 4 * 1024 * 1024);
+	draw_stage = QFV_CreateStagingBuffer (device, 4 * 1024 * 1024, 4,
+										  ctx->cmdpool);
 	conchars_sampler = QFV_GetSampler (ctx, "quakepic");
 
 	qpic_t     *charspic = Draw_Font8x8Pic ();
@@ -604,9 +583,8 @@ Vulkan_DrawReset (vulkan_ctx_t *ctx)
 void
 Vulkan_FlushText (vulkan_ctx_t *ctx)
 {
-	if (draw_stage->offset) {
-		flush_draw_scrap (ctx);
-	}
+	flush_draw_scrap (ctx);
+
 	qfv_device_t *device = ctx->device;
 	qfv_devfuncs_t *dfunc = device->funcs;
 	__auto_type frame = &ctx->framebuffers.a[ctx->curFrame];

libs/video/renderer/vulkan/vulkan_vid_common.c

@@ -203,8 +203,10 @@ Vulkan_CreateDevice (vulkan_ctx_t *ctx)
 void
 Vulkan_CreateStagingBuffers (vulkan_ctx_t *ctx)
 {
-	ctx->staging[0] = QFV_CreateStagingBuffer (ctx->device, 1024*1024);
-	ctx->staging[1] = QFV_CreateStagingBuffer (ctx->device, 1024*1024);
+	ctx->staging[0] = QFV_CreateStagingBuffer (ctx->device, 1024*1024, 1,
+											   ctx->cmdpool);
+	ctx->staging[1] = QFV_CreateStagingBuffer (ctx->device, 1024*1024, 1,
+											   ctx->cmdpool);
 }
 
 void